MCO-2599: makes bootc-operator aware of karpenter

api/v1alpha1/constants.go

@@ -30,4 +30,20 @@ const (
 	// state for which the controller considered emitting an event. It is
 	// observability bookkeeping only and is never used to drive reconciliation.
 	AnnotationLastObservedState = "bootc.dev/last-observed-state"
+
+	// AnnotationOwnsKarpenterDoNotRepair is set on a BootcNode to record that
+	// the controller added the karpenter.sh/do-not-repair annotation to the K8s
+	// Node during slot assignment. When present, the controller removes the
+	// karpenter.sh/do-not-repair annotation from the Node when the slot is freed.
+	// It is not set when the annotation was already present before slot assignment
+	// (externally set), so the controller never removes an annotation it did not add.
+	AnnotationOwnsKarpenterDoNotRepair = "bootc.dev/owns-karpenter-do-not-repair"
+
+	// AnnotationOwnsKarpenterDoNotDisrupt is set on a BootcNode to record that
+	// the controller added the karpenter.sh/do-not-disrupt annotation to the K8s
+	// Node during slot assignment. When present, the controller removes the
+	// karpenter.sh/do-not-disrupt annotation from the Node when the slot is freed.
+	// It is not set when the annotation was already present before slot assignment
+	// (externally set), so the controller never removes an annotation it did not add.
+	AnnotationOwnsKarpenterDoNotDisrupt = "bootc.dev/owns-karpenter-do-not-disrupt"
 )

docs/src/SUMMARY.md

@@ -8,3 +8,4 @@
 - [Helm Chart](helm.md)
 - [Operations](operations/index.md)
   - [Managing a pool](operations/pool.md)
+  - [Karpenter integration](operations/karpenter.md)

docs/src/operations/karpenter.md

@@ -0,0 +1,99 @@
+# Karpenter integration
+
+bootc-operator is aware of [Karpenter](https://karpenter.sh/) and coordinates
+with it during node updates to prevent unnecessary node replacement or
+voluntary disruption.
+
+## Why this matters
+
+Karpenter monitors nodes for health and may decide to replace or disrupt a
+node that appears unhealthy or underutilized — for example, one that is
+cordoned, being drained, or not yet Ready after a reboot. All of those
+conditions arise naturally during a bootc-operator update window, so without
+coordination Karpenter could race with the operator and terminate a node that
+is in the middle of a legitimate OS update.
+
+Karpenter respects two node-level annotations:
+
+- `karpenter.sh/do-not-repair: "true"` — prevents Node Auto Repair from
+  replacing a node that has unhealthy conditions.
+- `karpenter.sh/do-not-disrupt: "true"` — prevents voluntary disruption
+  actions (consolidation, drift) from selecting this node.
+
+> **Note:** `karpenter.sh/do-not-repair` was introduced in
+> [kubernetes-sigs/karpenter#3311](https://github.com/kubernetes-sigs/karpenter/pull/3311)
+> and has not yet appeared in a Karpenter release. This doc will be updated
+> with a minimum required version once one is available.
+
+bootc-operator sets both annotations automatically on Karpenter-managed nodes
+for the duration of the reboot slot.
+
+## How it works
+
+When the controller assigns a reboot slot to a node it checks whether the K8s
+Node object has an owner reference pointing to a Karpenter `NodeClaim`
+(`apiVersion: karpenter.sh/v1`, `kind: NodeClaim`). If it does, the
+controller takes the following steps:
+
+1. For each annotation (`do-not-repair`, `do-not-disrupt`): if the annotation
+   is **not** already present on the Node, the controller adds it (value
+   `"true"`) and records ownership by setting the corresponding tracking
+   annotation on the `BootcNode` object:
+   - `bootc.dev/owns-karpenter-do-not-repair: "true"`
+   - `bootc.dev/owns-karpenter-do-not-disrupt: "true"`
+2. The node proceeds through the normal update flow: drain, reboot, image
+   verification.
+3. When the slot is freed (node is Ready and running the new image), the
+   controller checks the tracking annotations on the `BootcNode`. For each
+   annotation it owns, it removes the corresponding annotation from the Node
+   and clears the tracking annotation from the `BootcNode`. Both removals are
+   issued as a single Node patch.
+
+## Pre-existing annotations
+
+If either Karpenter annotation is already on the Node before slot assignment —
+set by an external actor, another controller, or a manual operation —
+bootc-operator leaves it completely alone:
+
+- It does **not** overwrite or change the annotation on assignment.
+- It does **not** set the corresponding tracking annotation on the `BootcNode`.
+- It does **not** remove the annotation when the slot is freed.
+
+The two annotations are tracked independently. If only one is pre-existing, the
+operator owns (and later removes) only the other.
+
+This ensures bootc-operator never removes an annotation it did not add.
+
+## Limitations
+
+`karpenter.sh/do-not-disrupt` only blocks voluntary disruption methods
+(consolidation and drift). It does not protect a node against forceful
+termination triggered by expiration (`expireAfter`), interruption (spot
+instance reclaim), Node Auto Repair, or manual deletion. If any of those
+events occur while a node is in a bootc-operator reboot slot, Karpenter may
+still terminate it.
+
+`karpenter.sh/do-not-repair` similarly has no effect on voluntary disruption
+or expiration — it only suppresses Node Auto Repair.
+
+If a node is terminated by expiration, interruption, or any other forceful
+method while a bootc-operator update is in progress, the termination is caused
+by Karpenter acting on its own policy — not by the bootc-operator update. The
+update simply happened to be in flight at the time.
+
+Refer to the [Karpenter disruption documentation](https://karpenter.sh/docs/concepts/disruption/)
+for a full breakdown of which controls apply to which disruption methods.
+
+## Non-Karpenter nodes
+
+Nodes without a `NodeClaim` owner reference are unaffected. The feature
+activates only when the owner reference check matches.
+
+## Annotation reference
+
+| Annotation | Object | Managed by | Purpose |
+|---|---|---|---|
+| `karpenter.sh/do-not-repair` | K8s Node | bootc-operator (or external) | Prevents Karpenter Node Auto Repair from replacing this node during an update |
+| `karpenter.sh/do-not-disrupt` | K8s Node | bootc-operator (or external) | Prevents Karpenter voluntary disruption (consolidation, drift) during an update |
+| `bootc.dev/owns-karpenter-do-not-repair` | BootcNode | bootc-operator | Tracks that the operator added `do-not-repair`; cleared when the annotation is removed on slot free |
+| `bootc.dev/owns-karpenter-do-not-disrupt` | BootcNode | bootc-operator | Tracks that the operator added `do-not-disrupt`; cleared when the annotation is removed on slot free |

internal/controller/rollout.go

@@ -34,6 +34,18 @@ import (
 // https://github.com/bootc-dev/bootc-operator/issues/99.
 const unhealthySlotHaltThreshold = 2
 
+// Karpenter integration constants. The controller checks for a NodeClaim owner
+// reference and manages karpenter.sh/do-not-repair and
+// karpenter.sh/do-not-disrupt to prevent Karpenter from replacing or
+// voluntarily disrupting a node while it is being updated.
+// See docs/src/operations/karpenter.md.
+const (
+	karpenterDoNotRepairAnnotationKey  = "karpenter.sh/do-not-repair"
+	karpenterDoNotDisruptAnnotationKey = "karpenter.sh/do-not-disrupt"
+	karpenterNodeClaimAPIVersion       = "karpenter.sh/v1"
+	karpenterNodeClaimKind             = "NodeClaim"
+)
+
 // rolloutState holds the classified BootcNodes for a single reconcile
 // pass.
 type rolloutState struct {
@@ -188,6 +200,17 @@ func (r *BootcNodePoolReconciler) driveRollout(
 	return rs, nil
 }
 
+// nodeIsOwnedByKarpenterNodeClaim reports whether the K8s Node has an owner
+// reference pointing to a Karpenter NodeClaim (karpenter.sh/v1).
+func nodeIsOwnedByKarpenterNodeClaim(node *corev1.Node) bool {
+	for _, ref := range node.OwnerReferences {
+		if ref.APIVersion == karpenterNodeClaimAPIVersion && ref.Kind == karpenterNodeClaimKind {
+			return true
+		}
+	}
+	return false
+}
+
 // assignRebootSlot marks a BootcNode as occupying a reboot slot and
 // cordons the corresponding K8s Node. It sets the in-reboot-slot
 // annotation on the BootcNode, records prior cordon state in the
@@ -200,7 +223,18 @@ func (r *BootcNodePoolReconciler) assignRebootSlot(
 ) error {
 	log := logf.FromContext(ctx)
 
-	// Set annotations on the BootcNode if not already present.
+	// Determine Karpenter intent once so both patch blocks stay consistent.
+	// shouldOwn* is true when the node is Karpenter-managed and the respective
+	// annotation is not already present (set by an external actor). In that
+	// case, the controller adds the annotation and records ownership so it can
+	// be cleaned up on slot free.
+	karpenterOwned := nodeIsOwnedByKarpenterNodeClaim(node)
+	shouldOwnDoNotRepair := karpenterOwned &&
+		!metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotRepairAnnotationKey)
+	shouldOwnDoNotDisrupt := karpenterOwned &&
+		!metav1.HasAnnotation(node.ObjectMeta, karpenterDoNotDisruptAnnotationKey)
+
+	// Patch the BootcNode with slot annotations if not already present.
 	if !metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) {
 		log.Info("Assigning reboot slot", "node", bn.Name)
 		modifiedBN := bn.DeepCopy()
@@ -214,19 +248,44 @@ func (r *BootcNodePoolReconciler) assignRebootSlot(
 		} else {
 			modifiedBN.Annotations[bootcv1alpha1.AnnotationWasCordoned] = "false"
 		}
+		// Track Karpenter annotation ownership so freeRebootSlot knows
+		// whether to remove the annotations from the Node.
+		if shouldOwnDoNotRepair {
+			modifiedBN.Annotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair] = "true"
+		}
+		if shouldOwnDoNotDisrupt {
+			modifiedBN.Annotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt] = "true"
+		}
 		if err := r.Patch(ctx, modifiedBN, client.MergeFrom(bn)); err != nil {
 			return fmt.Errorf("annotating BootcNode: %w", err)
 		}
 		*bn = *modifiedBN
 	}
 
-	// Cordon the K8s Node if not already cordoned.
-	if !node.Spec.Unschedulable {
-		log.Info("Cordoning node", "node", node.Name)
+	// Patch the K8s Node: cordon it and, when we own the Karpenter
+	// annotations, set them. All are idempotent; we combine them into a
+	// single patch to minimise API calls.
+	if !node.Spec.Unschedulable || shouldOwnDoNotRepair || shouldOwnDoNotDisrupt {
 		modifiedNode := node.DeepCopy()
-		modifiedNode.Spec.Unschedulable = true
+		if !node.Spec.Unschedulable {
+			log.Info("Cordoning node", "node", node.Name)
+			modifiedNode.Spec.Unschedulable = true
+		}
+		if shouldOwnDoNotRepair || shouldOwnDoNotDisrupt {
+			if modifiedNode.Annotations == nil {
+				modifiedNode.Annotations = map[string]string{}
+			}
+			if shouldOwnDoNotRepair {
+				log.Info("Setting karpenter.sh/do-not-repair on node", "node", node.Name)
+				modifiedNode.Annotations[karpenterDoNotRepairAnnotationKey] = "true"
+			}
+			if shouldOwnDoNotDisrupt {
+				log.Info("Setting karpenter.sh/do-not-disrupt on node", "node", node.Name)
+				modifiedNode.Annotations[karpenterDoNotDisruptAnnotationKey] = "true"
+			}
+		}
 		if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil {
-			return fmt.Errorf("cordoning node: %w", err)
+			return fmt.Errorf("patching node: %w", err)
 		}
 		*node = *modifiedNode
 	}
@@ -272,12 +331,46 @@ func (r *BootcNodePoolReconciler) freeRebootSlot(
 	bn *bootcv1alpha1.BootcNode,
 	node *corev1.Node,
 ) error {
+	log := logf.FromContext(ctx)
+
+	// Remove karpenter.sh/do-not-repair and karpenter.sh/do-not-disrupt from
+	// the Node if the controller added them. We check the tracking annotations
+	// on the BootcNode rather than inspecting the Node directly, so an external
+	// actor who later removes the annotations doesn't cause spurious patches.
+	// Both removals are combined into a single Node patch when needed.
+	ownsRepair := metav1.HasAnnotation(
+		bn.ObjectMeta,
+		bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair,
+	)
+	ownsDisrupt := metav1.HasAnnotation(
+		bn.ObjectMeta,
+		bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt,
+	)
+	if ownsRepair || ownsDisrupt {
+		modifiedNode := node.DeepCopy()
+		if ownsRepair {
+			log.Info("Removing karpenter.sh/do-not-repair from node", "node", node.Name)
+			delete(modifiedNode.Annotations, karpenterDoNotRepairAnnotationKey)
+		}
+		if ownsDisrupt {
+			log.Info("Removing karpenter.sh/do-not-disrupt from node", "node", node.Name)
+			delete(modifiedNode.Annotations, karpenterDoNotDisruptAnnotationKey)
+		}
+		if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil {
+			return fmt.Errorf("removing karpenter annotations from node: %w", err)
+		}
+		*node = *modifiedNode
+	}
+
 	if err := r.restoreCordonState(ctx, bn, node); err != nil {
 		return err
 	}
 
-	// Remove both reboot slot annotations from the BootcNode.
+	// Remove all reboot slot annotations from the BootcNode, including the
+	// Karpenter ownership trackers if present.
 	modified := bn.DeepCopy()
+	delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)
+	delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)
 	delete(modified.Annotations, bootcv1alpha1.AnnotationWasCordoned)
 	delete(modified.Annotations, bootcv1alpha1.AnnotationInRebootSlot)
 	if err := r.Patch(ctx, modified, client.MergeFrom(bn)); err != nil {