diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go index d3d3338c..1c6e4879 100644 --- a/api/v1alpha1/constants.go +++ b/api/v1alpha1/constants.go @@ -30,4 +30,20 @@ const ( // state for which the controller considered emitting an event. It is // observability bookkeeping only and is never used to drive reconciliation. AnnotationLastObservedState = "bootc.dev/last-observed-state" + + // AnnotationOwnsKarpenterDoNotRepair is set on a BootcNode to record that + // the controller added the karpenter.sh/do-not-repair annotation to the K8s + // Node during slot assignment. When present, the controller removes the + // karpenter.sh/do-not-repair annotation from the Node when the slot is freed. + // It is not set when the annotation was already present before slot assignment + // (externally set), so the controller never removes an annotation it did not add. + AnnotationOwnsKarpenterDoNotRepair = "bootc.dev/owns-karpenter-do-not-repair" + + // AnnotationOwnsKarpenterDoNotDisrupt is set on a BootcNode to record that + // the controller added the karpenter.sh/do-not-disrupt annotation to the K8s + // Node during slot assignment. When present, the controller removes the + // karpenter.sh/do-not-disrupt annotation from the Node when the slot is freed. + // It is not set when the annotation was already present before slot assignment + // (externally set), so the controller never removes an annotation it did not add. + AnnotationOwnsKarpenterDoNotDisrupt = "bootc.dev/owns-karpenter-do-not-disrupt" ) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index fbfaf433..d411f43b 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -8,3 +8,4 @@ - [Helm Chart](helm.md) - [Operations](operations/index.md) - [Managing a pool](operations/pool.md) + - [Karpenter integration](operations/karpenter.md) diff --git a/docs/src/operations/karpenter.md b/docs/src/operations/karpenter.md new file mode 100644 index 00000000..330aa55c --- /dev/null +++ b/docs/src/operations/karpenter.md @@ -0,0 +1,99 @@ +# Karpenter integration + +bootc-operator is aware of [Karpenter](https://karpenter.sh/) and coordinates +with it during node updates to prevent unnecessary node replacement or +voluntary disruption. + +## Why this matters + +Karpenter monitors nodes for health and may decide to replace or disrupt a +node that appears unhealthy or underutilized — for example, one that is +cordoned, being drained, or not yet Ready after a reboot. All of those +conditions arise naturally during a bootc-operator update window, so without +coordination Karpenter could race with the operator and terminate a node that +is in the middle of a legitimate OS update. + +Karpenter respects two node-level annotations: + +- `karpenter.sh/do-not-repair: "true"` — prevents Node Auto Repair from + replacing a node that has unhealthy conditions. +- `karpenter.sh/do-not-disrupt: "true"` — prevents voluntary disruption + actions (consolidation, drift) from selecting this node. + +> **Note:** `karpenter.sh/do-not-repair` was introduced in +> [kubernetes-sigs/karpenter#3311](https://github.com/kubernetes-sigs/karpenter/pull/3311) +> and has not yet appeared in a Karpenter release. This doc will be updated +> with a minimum required version once one is available. + +bootc-operator sets both annotations automatically on Karpenter-managed nodes +for the duration of the reboot slot. + +## How it works + +When the controller assigns a reboot slot to a node it checks whether the K8s +Node object has an owner reference pointing to a Karpenter `NodeClaim` +(`apiVersion: karpenter.sh/v1`, `kind: NodeClaim`). If it does, the +controller takes the following steps: + +1. For each annotation (`do-not-repair`, `do-not-disrupt`): if the annotation + is **not** already present on the Node, the controller adds it (value + `"true"`) and records ownership by setting the corresponding tracking + annotation on the `BootcNode` object: + - `bootc.dev/owns-karpenter-do-not-repair: "true"` + - `bootc.dev/owns-karpenter-do-not-disrupt: "true"` +2. The node proceeds through the normal update flow: drain, reboot, image + verification. +3. When the slot is freed (node is Ready and running the new image), the + controller checks the tracking annotations on the `BootcNode`. For each + annotation it owns, it removes the corresponding annotation from the Node + and clears the tracking annotation from the `BootcNode`. Both removals are + issued as a single Node patch. + +## Pre-existing annotations + +If either Karpenter annotation is already on the Node before slot assignment — +set by an external actor, another controller, or a manual operation — +bootc-operator leaves it completely alone: + +- It does **not** overwrite or change the annotation on assignment. +- It does **not** set the corresponding tracking annotation on the `BootcNode`. +- It does **not** remove the annotation when the slot is freed. + +The two annotations are tracked independently. If only one is pre-existing, the +operator owns (and later removes) only the other. + +This ensures bootc-operator never removes an annotation it did not add. + +## Limitations + +`karpenter.sh/do-not-disrupt` only blocks voluntary disruption methods +(consolidation and drift). It does not protect a node against forceful +termination triggered by expiration (`expireAfter`), interruption (spot +instance reclaim), Node Auto Repair, or manual deletion. If any of those +events occur while a node is in a bootc-operator reboot slot, Karpenter may +still terminate it. + +`karpenter.sh/do-not-repair` similarly has no effect on voluntary disruption +or expiration — it only suppresses Node Auto Repair. + +If a node is terminated by expiration, interruption, or any other forceful +method while a bootc-operator update is in progress, the termination is caused +by Karpenter acting on its own policy — not by the bootc-operator update. The +update simply happened to be in flight at the time. + +Refer to the [Karpenter disruption documentation](https://karpenter.sh/docs/concepts/disruption/) +for a full breakdown of which controls apply to which disruption methods. + +## Non-Karpenter nodes + +Nodes without a `NodeClaim` owner reference are unaffected. The feature +activates only when the owner reference check matches. + +## Annotation reference + +| Annotation | Object | Managed by | Purpose | +|---|---|---|---| +| `karpenter.sh/do-not-repair` | K8s Node | bootc-operator (or external) | Prevents Karpenter Node Auto Repair from replacing this node during an update | +| `karpenter.sh/do-not-disrupt` | K8s Node | bootc-operator (or external) | Prevents Karpenter voluntary disruption (consolidation, drift) during an update | +| `bootc.dev/owns-karpenter-do-not-repair` | BootcNode | bootc-operator | Tracks that the operator added `do-not-repair`; cleared when the annotation is removed on slot free | +| `bootc.dev/owns-karpenter-do-not-disrupt` | BootcNode | bootc-operator | Tracks that the operator added `do-not-disrupt`; cleared when the annotation is removed on slot free | diff --git a/internal/controller/rollout.go b/internal/controller/rollout.go index f1c3d069..402a1176 100644 --- a/internal/controller/rollout.go +++ b/internal/controller/rollout.go @@ -34,6 +34,18 @@ import ( // https://github.com/bootc-dev/bootc-operator/issues/99. const unhealthySlotHaltThreshold = 2 +// Karpenter integration constants. The controller checks for a NodeClaim owner +// reference and manages karpenter.sh/do-not-repair and +// karpenter.sh/do-not-disrupt to prevent Karpenter from replacing or +// voluntarily disrupting a node while it is being updated. +// See docs/src/operations/karpenter.md. +const ( + karpenterDoNotRepairAnnotationKey = "karpenter.sh/do-not-repair" + karpenterDoNotDisruptAnnotationKey = "karpenter.sh/do-not-disrupt" + karpenterNodeClaimAPIVersion = "karpenter.sh/v1" + karpenterNodeClaimKind = "NodeClaim" +) + // rolloutState holds the classified BootcNodes for a single reconcile // pass. type rolloutState struct { @@ -188,6 +200,17 @@ func (r *BootcNodePoolReconciler) driveRollout( return rs, nil } +// nodeIsOwnedByKarpenterNodeClaim reports whether the K8s Node has an owner +// reference pointing to a Karpenter NodeClaim (karpenter.sh/v1). +func nodeIsOwnedByKarpenterNodeClaim(node *corev1.Node) bool { + for _, ref := range node.OwnerReferences { + if ref.APIVersion == karpenterNodeClaimAPIVersion && ref.Kind == karpenterNodeClaimKind { + return true + } + } + return false +} + // assignRebootSlot marks a BootcNode as occupying a reboot slot and // cordons the corresponding K8s Node. It sets the in-reboot-slot // annotation on the BootcNode, records prior cordon state in the @@ -200,7 +223,18 @@ func (r *BootcNodePoolReconciler) assignRebootSlot( ) error { log := logf.FromContext(ctx) - // Set annotations on the BootcNode if not already present. + // Determine Karpenter intent once so both patch blocks stay consistent. + // shouldOwn* is true when the node is Karpenter-managed and the respective + // annotation is not already present (set by an external actor). In that + // case, the controller adds the annotation and records ownership so it can + // be cleaned up on slot free. + karpenterOwned := nodeIsOwnedByKarpenterNodeClaim(node) + shouldOwnDoNotRepair := karpenterOwned && + !metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotRepairAnnotationKey) + shouldOwnDoNotDisrupt := karpenterOwned && + !metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotDisruptAnnotationKey) + + // Patch the BootcNode with slot annotations if not already present. if !metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { log.Info("Assigning reboot slot", "node", bn.Name) modifiedBN := bn.DeepCopy() @@ -214,19 +248,44 @@ func (r *BootcNodePoolReconciler) assignRebootSlot( } else { modifiedBN.Annotations[bootcv1alpha1.AnnotationWasCordoned] = "false" } + // Track Karpenter annotation ownership so freeRebootSlot knows + // whether to remove the annotations from the Node. + if shouldOwnDoNotRepair { + modifiedBN.Annotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair] = "true" + } + if shouldOwnDoNotDisrupt { + modifiedBN.Annotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt] = "true" + } if err := r.Patch(ctx, modifiedBN, client.MergeFrom(bn)); err != nil { return fmt.Errorf("annotating BootcNode: %w", err) } *bn = *modifiedBN } - // Cordon the K8s Node if not already cordoned. - if !node.Spec.Unschedulable { - log.Info("Cordoning node", "node", node.Name) + // Patch the K8s Node: cordon it and, when we own the Karpenter + // annotations, set them. All are idempotent; we combine them into a + // single patch to minimise API calls. + if !node.Spec.Unschedulable || shouldOwnDoNotRepair || shouldOwnDoNotDisrupt { modifiedNode := node.DeepCopy() - modifiedNode.Spec.Unschedulable = true + if !node.Spec.Unschedulable { + log.Info("Cordoning node", "node", node.Name) + modifiedNode.Spec.Unschedulable = true + } + if shouldOwnDoNotRepair || shouldOwnDoNotDisrupt { + if modifiedNode.Annotations == nil { + modifiedNode.Annotations = map[string]string{} + } + if shouldOwnDoNotRepair { + log.Info("Setting karpenter.sh/do-not-repair on node", "node", node.Name) + modifiedNode.Annotations[karpenterDoNotRepairAnnotationKey] = "true" + } + if shouldOwnDoNotDisrupt { + log.Info("Setting karpenter.sh/do-not-disrupt on node", "node", node.Name) + modifiedNode.Annotations[karpenterDoNotDisruptAnnotationKey] = "true" + } + } if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil { - return fmt.Errorf("cordoning node: %w", err) + return fmt.Errorf("patching node: %w", err) } *node = *modifiedNode } @@ -272,12 +331,46 @@ func (r *BootcNodePoolReconciler) freeRebootSlot( bn *bootcv1alpha1.BootcNode, node *corev1.Node, ) error { + log := logf.FromContext(ctx) + + // Remove karpenter.sh/do-not-repair and karpenter.sh/do-not-disrupt from + // the Node if the controller added them. We check the tracking annotations + // on the BootcNode rather than inspecting the Node directly, so an external + // actor who later removes the annotations doesn't cause spurious patches. + // Both removals are combined into a single Node patch when needed. + ownsRepair := metav1.HasAnnotation( + bn.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair, + ) + ownsDisrupt := metav1.HasAnnotation( + bn.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt, + ) + if ownsRepair || ownsDisrupt { + modifiedNode := node.DeepCopy() + if ownsRepair { + log.Info("Removing karpenter.sh/do-not-repair from node", "node", node.Name) + delete(modifiedNode.Annotations, karpenterDoNotRepairAnnotationKey) + } + if ownsDisrupt { + log.Info("Removing karpenter.sh/do-not-disrupt from node", "node", node.Name) + delete(modifiedNode.Annotations, karpenterDoNotDisruptAnnotationKey) + } + if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil { + return fmt.Errorf("removing karpenter annotations from node: %w", err) + } + *node = *modifiedNode + } + if err := r.restoreCordonState(ctx, bn, node); err != nil { return err } - // Remove both reboot slot annotations from the BootcNode. + // Remove all reboot slot annotations from the BootcNode, including the + // Karpenter ownership trackers if present. modified := bn.DeepCopy() + delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair) + delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt) delete(modified.Annotations, bootcv1alpha1.AnnotationWasCordoned) delete(modified.Annotations, bootcv1alpha1.AnnotationInRebootSlot) if err := r.Patch(ctx, modified, client.MergeFrom(bn)); err != nil { diff --git a/internal/controller/rollout_karpenter_envtest_test.go b/internal/controller/rollout_karpenter_envtest_test.go new file mode 100644 index 00000000..91c25755 --- /dev/null +++ b/internal/controller/rollout_karpenter_envtest_test.go @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +// TestKarpenterAnnotations is a table-driven integration test that covers +// Karpenter NodeClaim annotation management across all distinct scenarios. Each +// row exercises a different combination of node ownership, pre-existing +// annotations, and cordon state. The focused single-scenario predecessors of +// this test are preserved in rollout_karpenter_focused_envtest_test.go. + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +func TestKarpenterAnnotations(t *testing.T) { + tests := []struct { + // name is the subtest name and is used to derive unique resource names. + name string + + // Node setup. + nodeClaim bool // whether to attach a NodeClaim owner reference + preExistingAnnotations map[string]string // annotations to set on the Node before the rollout + preCordon bool // whether the Node starts cordoned + + // Expected BootcNode ownership annotations while the reboot slot is held. + wantOwnsRepair bool // AnnotationOwnsKarpenterDoNotRepair should be present + wantOwnsDisrupt bool // AnnotationOwnsKarpenterDoNotDisrupt should be present + + // Expected Node annotations while the reboot slot is held. + wantRepairDuring bool // karpenter.sh/do-not-repair should be present + wantDisruptDuring bool // karpenter.sh/do-not-disrupt should be present + + // Expected Node annotations after the slot is freed. + wantRepairAfter bool // karpenter.sh/do-not-repair should still be present + wantDisruptAfter bool // karpenter.sh/do-not-disrupt should still be present + + // Expected Node cordon state after the slot is freed. + wantCordonedAfter bool + }{ + // A plain NodeClaim-owned node with no pre-existing Karpenter state. + // The operator adds both annotations on slot assignment, records + // ownership, and removes them when the slot is freed. + { + name: "nodeclaim owned, no pre-existing annotations", + nodeClaim: true, + wantOwnsRepair: true, + wantOwnsDisrupt: true, + wantRepairDuring: true, + wantDisruptDuring: true, + wantRepairAfter: false, + wantDisruptAfter: false, + wantCordonedAfter: false, + }, + // A node with both Karpenter annotations already set by an external + // actor and no NodeClaim owner reference. The operator must not claim + // ownership of either annotation and must leave both untouched + // throughout, including after the slot is freed. + { + name: "no nodeclaim, both annotations pre-existing", + nodeClaim: false, + preExistingAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantOwnsRepair: false, + wantOwnsDisrupt: false, + wantRepairDuring: true, // preserved as-is + wantDisruptDuring: true, // preserved as-is + wantRepairAfter: true, // untouched by operator + wantDisruptAfter: true, // untouched by operator + wantCordonedAfter: false, + }, + // A NodeClaim-owned node that was already cordoned before the rollout. + // The operator adds both Karpenter annotations and removes them on slot + // free, but the pre-existing cordon must be preserved throughout. + { + name: "nodeclaim owned, pre-cordoned", + nodeClaim: true, + preCordon: true, + wantOwnsRepair: true, + wantOwnsDisrupt: true, + wantRepairDuring: true, + wantDisruptDuring: true, + wantRepairAfter: false, + wantDisruptAfter: false, + wantCordonedAfter: true, // pre-existing cordon preserved + }, + // A NodeClaim-owned node where only do-not-repair is pre-existing. + // The operator must not claim do-not-repair (already set externally) + // but must add, own, and later remove do-not-disrupt independently. + // This verifies that per-annotation ownership tracking is applied + // independently rather than as an all-or-nothing decision. + { + name: "nodeclaim owned, only do-not-repair pre-existing", + nodeClaim: true, + preExistingAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + }, + wantOwnsRepair: false, // pre-existing; operator does not claim it + wantOwnsDisrupt: true, // not pre-existing; operator adds and owns it + wantRepairDuring: true, // preserved from pre-existing + wantDisruptDuring: true, // added by operator + wantRepairAfter: true, // not owned; left in place + wantDisruptAfter: false, // owned; removed by operator + wantCordonedAfter: false, + }, + } + + for i, tc := range tests { + tc := tc + // Derive unique, collision-free resource names from the subtest index. + nodeName := fmt.Sprintf("karp-ann-%d-node", i) + poolName := fmt.Sprintf("karp-ann-%d-pool", i) + + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + // Build and create the K8s Node. + node := testutil.NewK8sNode(nodeName, testutil.WorkerLabels()) + if tc.nodeClaim { + node.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: nodeName + "-claim", + UID: types.UID(nodeName + "-claim-uid"), + }} + } + if len(tc.preExistingAnnotations) > 0 { + node.Annotations = tc.preExistingAnnotations + } + if tc.preCordon { + node.Spec.Unschedulable = true + } + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, node) }) + + // Create pool and wait for BootcNode. + pool := testutil.NewPool(poolName, testImageDigestRefB, + testutil.WithWorkerSelector(), + testutil.WithMaxUnavailable(intstr.FromInt32(1)), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(func() error { + return k8sClient.Get( + ctx, + client.ObjectKey{Name: nodeName}, + &bootcv1alpha1.BootcNode{}, + ) + }).Should(Succeed()) + + simulateDaemonStatus(g, ctx, nodeName, testDigestA, bootcv1alpha1.NodeReasonStaged) + + // Wait for slot assignment and drain completion. + g.Eventually(func() (*bootcv1alpha1.BootcNode, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return &bn, err + }).Should(And( + HaveField("Annotations", HaveKey(bootcv1alpha1.AnnotationInRebootSlot)), + HaveField("Spec.DesiredImageState", Equal(bootcv1alpha1.DesiredImageStateBooted)), + ), "BootcNode should have reboot slot and desiredImageState Booted") + + // Assert ownership annotations on BootcNode. + g.Eventually(func() (map[string]string, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Annotations, err + }).Should(matchOwnership(tc.wantOwnsRepair, tc.wantOwnsDisrupt), + "BootcNode ownership annotations during slot") + + // Assert Karpenter annotations on Node. + g.Eventually(func() (map[string]string, error) { + var n corev1.Node + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n) + return n.Annotations, err + }).Should(matchKarpenterAnnotations(tc.wantRepairDuring, tc.wantDisruptDuring), + "Node Karpenter annotations during slot") + + // Simulate successful reboot and node becoming Ready. + simulateDaemonStatus(g, ctx, nodeName, testDigestB, bootcv1alpha1.NodeReasonIdle) + setNodeReady(g, ctx, nodeName) + + // Wait for slot to be freed. + g.Eventually(func() (map[string]string, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Annotations, err + }).Should(And( + Not(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)), + Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)), + Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)), + ), "BootcNode should have no slot or ownership annotations after slot free") + + // Assert final Node annotation state. + g.Eventually(func() (map[string]string, error) { + var n corev1.Node + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n) + return n.Annotations, err + }).Should(matchKarpenterAnnotations(tc.wantRepairAfter, tc.wantDisruptAfter), + "Node Karpenter annotations after slot free") + + // Assert final cordon state. + g.Eventually(func() (bool, error) { + var n corev1.Node + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n) + return n.Spec.Unschedulable, err + }).Should(Equal(tc.wantCordonedAfter), "Node cordon state after slot free") + }) + } +} + +// matchOwnership returns a Gomega matcher for the BootcNode annotation map +// that asserts the presence or absence of the two Karpenter ownership +// annotations independently. +func matchOwnership(wantRepair, wantDisrupt bool) OmegaMatcher { + repair := Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + if wantRepair { + repair = HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair) + } + disrupt := Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + if wantDisrupt { + disrupt = HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt) + } + return And(repair, disrupt) +} + +// matchKarpenterAnnotations returns a Gomega matcher for the Node annotation +// map that asserts the presence or absence of the two Karpenter annotations +// independently. +func matchKarpenterAnnotations(wantRepair, wantDisrupt bool) OmegaMatcher { + repair := Not(HaveKey(karpenterDoNotRepairAnnotationKey)) + if wantRepair { + repair = HaveKey(karpenterDoNotRepairAnnotationKey) + } + disrupt := Not(HaveKey(karpenterDoNotDisruptAnnotationKey)) + if wantDisrupt { + disrupt = HaveKey(karpenterDoNotDisruptAnnotationKey) + } + return And(repair, disrupt) +}