From 56b65b8239bce75ef543c1aa5af390bdb64f9006 Mon Sep 17 00:00:00 2001 From: Caleb Xu Date: Fri, 11 Sep 2026 14:30:56 -0400 Subject: [PATCH] feat: gate provisioning on platform readiness --- cmd/main.go | 5 +- config/rbac/role.yaml | 26 ++ .../controller/clusterinstance_controller.go | 76 ++++- internal/controller/platform_readiness.go | 169 +++++++++++ .../controller/platform_readiness_test.go | 281 ++++++++++++++++++ test/e2e/e2e_test.go | 17 ++ test/e2e/testdata/healthy-hco.yaml | 14 + test/e2e/testdata/vmi-crd.yaml | 21 ++ 8 files changed, 606 insertions(+), 3 deletions(-) create mode 100644 internal/controller/platform_readiness.go create mode 100644 internal/controller/platform_readiness_test.go create mode 100644 test/e2e/testdata/healthy-hco.yaml diff --git a/cmd/main.go b/cmd/main.go index a1546bf..6ec5755 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -220,8 +220,9 @@ func main() { os.Exit(1) } if err := (&controller.ClusterInstanceReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + APIReader: mgr.GetAPIReader(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterInstance") os.Exit(1) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 170c221..a52fc3b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -36,6 +36,18 @@ rules: - patch - update - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get +- apiGroups: + - apps + resources: + - deployments + verbs: + - get - apiGroups: - batch resources: @@ -109,6 +121,13 @@ rules: - get - patch - update +- apiGroups: + - hco.kubevirt.io + resources: + - hyperconvergeds + verbs: + - get + - list - apiGroups: - hypershift.openshift.io resources: @@ -142,6 +161,13 @@ rules: - patch - update - watch +- apiGroups: + - multicluster.openshift.io + resources: + - multiclusterengines + verbs: + - get + - list - apiGroups: - route.openshift.io resources: diff --git a/internal/controller/clusterinstance_controller.go b/internal/controller/clusterinstance_controller.go index 51fbabc..a9080cd 100644 --- a/internal/controller/clusterinstance_controller.go +++ b/internal/controller/clusterinstance_controller.go @@ -23,6 +23,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -58,7 +59,8 @@ const ( // ClusterInstanceReconciler reconciles a ClusterInstance object type ClusterInstanceReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + APIReader client.Reader } // +kubebuilder:rbac:groups=guestcluster.opdev.io,resources=clusterinstances,verbs=get;list;watch;create;update;patch;delete @@ -80,6 +82,10 @@ type ClusterInstanceReconciler struct { // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=route.openshift.io,resources=routes/custom-host,verbs=create;update // +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get;list;watch +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get +// +kubebuilder:rbac:groups=hco.kubevirt.io,resources=hyperconvergeds,verbs=get;list +// +kubebuilder:rbac:groups=multicluster.openshift.io,resources=multiclusterengines,verbs=get;list +// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get // Reconcile drives a ClusterInstance through Provisioning -> Ready. It // delegates the actual creation and inspection of backing objects (KubeVirt @@ -142,6 +148,17 @@ func (r *ClusterInstanceReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } + if instance.Status.Phase != brokerv1alpha1.PhaseFailed && + (instance.Spec.Type == brokerv1alpha1.TopologyCRC || + (instance.Spec.Type == brokerv1alpha1.TopologyHCP && instance.Status.Phase != brokerv1alpha1.PhaseReady)) { + if result, err := r.gatePlatformOperations(ctx, instance); result != nil || err != nil { + if result == nil { + return ctrl.Result{}, err + } + return *result, err + } + } + if instance.Status.Phase == brokerv1alpha1.PhaseReady { if instance.Spec.Type == brokerv1alpha1.TopologyCRC { return r.reconcileReadyCRC(ctx, instance) @@ -159,6 +176,63 @@ func (r *ClusterInstanceReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } +func (r *ClusterInstanceReconciler) gatePlatformOperations(ctx context.Context, instance *brokerv1alpha1.ClusterInstance) (*ctrl.Result, error) { + previousStatus := instance.Status.DeepCopy() + conditions := []platformCondition{r.checkHyperConverged(ctx)} + if instance.Spec.Type == brokerv1alpha1.TopologyHCP { + conditions = append(conditions, r.checkMultiClusterEngine(ctx)) + } + + ready := true + for _, condition := range conditions { + condition.condition.ObservedGeneration = instance.Generation + apimeta.SetStatusCondition(&instance.Status.Conditions, condition.condition) + ready = ready && condition.ready + } + instance.Status.ObservedGeneration = instance.Generation + if !ready { + instance.Status.Phase = brokerv1alpha1.PhaseProvisioning + apimeta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: conditionTypeReady, + Status: metav1.ConditionFalse, + Reason: firstBlockedReason(conditions), + Message: "Platform dependencies are not ready", + ObservedGeneration: instance.Generation, + }) + if err := r.updatePlatformStatus(ctx, instance, previousStatus); err != nil { + return nil, err + } + return &ctrl.Result{RequeueAfter: requeueInterval}, nil + } + + if err := r.updatePlatformStatus(ctx, instance, previousStatus); err != nil { + return nil, err + } + if !equality.Semantic.DeepEqual(*previousStatus, instance.Status) { + return &ctrl.Result{}, nil + } + return nil, nil +} + +func firstBlockedReason(conditions []platformCondition) string { + for _, condition := range conditions { + if !condition.ready { + return condition.condition.Reason + } + } + return "OperandNotReady" +} + +func (r *ClusterInstanceReconciler) updatePlatformStatus(ctx context.Context, instance *brokerv1alpha1.ClusterInstance, previousStatus *brokerv1alpha1.ClusterInstanceStatus) error { + if equality.Semantic.DeepEqual(*previousStatus, instance.Status) { + return nil + } + if err := r.Status().Update(ctx, instance); err != nil { + return fmt.Errorf("updating platform readiness status: %w", err) + } + return nil +} + // reconcileLeaseRefProjection maintains Status.LeaseRef as a read-only, // derived mirror of whichever ClusterLease (if any) currently names this // instance via its own Status.InstanceRef, the single source of truth for diff --git a/internal/controller/platform_readiness.go b/internal/controller/platform_readiness.go new file mode 100644 index 0000000..7c4a0c5 --- /dev/null +++ b/internal/controller/platform_readiness.go @@ -0,0 +1,169 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + conditionTypeHyperConvergedReady = "HyperConvergedReady" + conditionTypeMultiClusterEngineReady = "MultiClusterEngineReady" + hyperShiftGroup = "hypershift.openshift.io" +) + +var ( + hyperConvergedGVK = schema.GroupVersionKind{Group: "hco.kubevirt.io", Version: "v1beta1", Kind: "HyperConverged"} + multiClusterEngineGVK = schema.GroupVersionKind{Group: "multicluster.openshift.io", Version: "v1", Kind: "MultiClusterEngine"} + customResourceDefinitionGVK = schema.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"} +) + +type platformCondition struct { + condition metav1.Condition + ready bool +} + +func (r *ClusterInstanceReconciler) platformReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +func (r *ClusterInstanceReconciler) checkHyperConverged(ctx context.Context) platformCondition { + operands := &unstructured.UnstructuredList{} + operands.SetGroupVersionKind(hyperConvergedGVK.GroupVersion().WithKind("HyperConvergedList")) + if err := r.platformReader().List(ctx, operands); err != nil { + return unavailablePlatformCondition(conditionTypeHyperConvergedReady, "HyperConverged") + } + if len(operands.Items) != 1 { + return singletonPlatformCondition(conditionTypeHyperConvergedReady, "HyperConverged", len(operands.Items)) + } + + operand := &operands.Items[0] + if observedGeneration, found, _ := unstructured.NestedInt64(operand.Object, "status", "observedGeneration"); !found || observedGeneration < operand.GetGeneration() { + return notReadyPlatformCondition(conditionTypeHyperConvergedReady, "StatusStale", "HyperConverged status does not observe its current generation") + } + for _, expected := range []struct { + typeName string + status metav1.ConditionStatus + }{ + {typeName: "Available", status: metav1.ConditionTrue}, + {typeName: "Progressing", status: metav1.ConditionFalse}, + {typeName: "Degraded", status: metav1.ConditionFalse}, + } { + actual, found := conditionStatus(operand, expected.typeName) + if !found || actual != expected.status { + return notReadyPlatformCondition(conditionTypeHyperConvergedReady, "OperandNotReady", fmt.Sprintf("HyperConverged condition %s must be %s", expected.typeName, expected.status)) + } + } + + return readyPlatformCondition(conditionTypeHyperConvergedReady, "HyperConverged is available") +} + +func (r *ClusterInstanceReconciler) checkMultiClusterEngine(ctx context.Context) platformCondition { + operands := &unstructured.UnstructuredList{} + operands.SetGroupVersionKind(multiClusterEngineGVK.GroupVersion().WithKind("MultiClusterEngineList")) + if err := r.platformReader().List(ctx, operands); err != nil { + return unavailablePlatformCondition(conditionTypeMultiClusterEngineReady, "MultiClusterEngine") + } + if len(operands.Items) != 1 { + return singletonPlatformCondition(conditionTypeMultiClusterEngineReady, "MultiClusterEngine", len(operands.Items)) + } + if !hyperShiftComponentEnabled(&operands.Items[0]) { + return notReadyPlatformCondition(conditionTypeMultiClusterEngineReady, "ComponentDisabled", "MultiClusterEngine does not enable the hypershift component") + } + + for _, name := range []string{"hostedclusters." + hyperShiftGroup, "nodepools." + hyperShiftGroup} { + crd := &unstructured.Unstructured{} + crd.SetGroupVersionKind(customResourceDefinitionGVK) + if err := r.platformReader().Get(ctx, client.ObjectKey{Name: name}, crd); err != nil { + return unavailablePlatformCondition(conditionTypeMultiClusterEngineReady, name) + } + if status, found := conditionStatus(crd, "Established"); !found || status != metav1.ConditionTrue { + return notReadyPlatformCondition(conditionTypeMultiClusterEngineReady, "OperandNotReady", fmt.Sprintf("CustomResourceDefinition %s is not established", name)) + } + } + + operator := &appsv1.Deployment{} + if err := r.platformReader().Get(ctx, client.ObjectKey{Namespace: "hypershift", Name: "operator"}, operator); err != nil { + return unavailablePlatformCondition(conditionTypeMultiClusterEngineReady, "hypershift/operator Deployment") + } + if operator.Status.AvailableReplicas < 1 { + return notReadyPlatformCondition(conditionTypeMultiClusterEngineReady, "OperandNotReady", "Deployment hypershift/operator has no available replicas") + } + + return readyPlatformCondition(conditionTypeMultiClusterEngineReady, "HyperShift APIs and operator are available") +} + +func conditionStatus(obj *unstructured.Unstructured, typeName string) (metav1.ConditionStatus, bool) { + conditions, found, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + if !found { + return "", false + } + for _, item := range conditions { + condition, ok := item.(map[string]interface{}) + if !ok || condition["type"] != typeName { + continue + } + status, ok := condition["status"].(string) + return metav1.ConditionStatus(status), ok + } + return "", false +} + +func hyperShiftComponentEnabled(mce *unstructured.Unstructured) bool { + components, found, _ := unstructured.NestedSlice(mce.Object, "spec", "overrides", "components") + if !found { + return false + } + for _, item := range components { + component, ok := item.(map[string]interface{}) + if !ok || component["name"] != "hypershift" { + continue + } + enabled, found, _ := unstructured.NestedBool(component, "enabled") + return !found || enabled + } + return false +} + +func readyPlatformCondition(conditionType, message string) platformCondition { + return platformCondition{ready: true, condition: metav1.Condition{Type: conditionType, Status: metav1.ConditionTrue, Reason: "OperandReady", Message: message}} +} + +func singletonPlatformCondition(conditionType, operand string, count int) platformCondition { + if count == 0 { + return notReadyPlatformCondition(conditionType, "OperandNotFound", fmt.Sprintf("no %s operand was found", operand)) + } + return notReadyPlatformCondition(conditionType, "MultipleOperandsFound", fmt.Sprintf("found %d %s operands; exactly one is required", count, operand)) +} + +func unavailablePlatformCondition(conditionType, operand string) platformCondition { + return notReadyPlatformCondition(conditionType, "APIUnavailable", fmt.Sprintf("cannot read %s", operand)) +} + +func notReadyPlatformCondition(conditionType, reason, message string) platformCondition { + return platformCondition{condition: metav1.Condition{Type: conditionType, Status: metav1.ConditionFalse, Reason: reason, Message: message}} +} diff --git a/internal/controller/platform_readiness_test.go b/internal/controller/platform_readiness_test.go new file mode 100644 index 0000000..c2ecf1d --- /dev/null +++ b/internal/controller/platform_readiness_test.go @@ -0,0 +1,281 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + "time" + + hyperv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + brokerv1alpha1 "github.com/caxu-rh/guestcluster-operator/api/v1alpha1" +) + +const ( + operandNotFoundReason = "OperandNotFound" + apiVersionField = "apiVersion" + kindField = "kind" + metadataField = "metadata" + nameField = "name" + statusField = "status" + conditionTypeField = "type" +) + +func TestCheckHyperConverged(t *testing.T) { + tests := []struct { + name string + objects []client.Object + wantReady bool + wantReason string + }{ + {name: "missing operand", wantReason: operandNotFoundReason}, + {name: "multiple operands", objects: []client.Object{healthyHCO("one"), healthyHCO("two")}, wantReason: "MultipleOperandsFound"}, + {name: "stale status", objects: []client.Object{hco("hco", 2, 1, "True", "False", "False")}, wantReason: "StatusStale"}, + {name: "degraded", objects: []client.Object{hco("hco", 1, 1, "True", "False", "True")}, wantReason: "OperandNotReady"}, + {name: "healthy", objects: []client.Object{healthyHCO("hco")}, wantReady: true, wantReason: "OperandReady"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newPlatformFakeClient(t, tt.objects...) + r := &ClusterInstanceReconciler{Client: c, APIReader: c} + got := r.checkHyperConverged(context.Background()) + if got.ready != tt.wantReady || got.condition.Reason != tt.wantReason { + t.Fatalf("checkHyperConverged() = ready %t, reason %q; want ready %t, reason %q", got.ready, got.condition.Reason, tt.wantReady, tt.wantReason) + } + }) + } +} + +func TestCheckMultiClusterEngine(t *testing.T) { + tests := []struct { + name string + objects []client.Object + wantReady bool + wantReason string + }{ + {name: "missing operand", wantReason: "OperandNotFound"}, + {name: "component disabled", objects: []client.Object{mce(false)}, wantReason: "ComponentDisabled"}, + {name: "missing API", objects: []client.Object{mce(true)}, wantReason: "APIUnavailable"}, + {name: "operator unavailable", objects: append(hyperShiftAPIs(), mce(true)), wantReason: "APIUnavailable"}, + {name: "aggregate progressing does not block", objects: append(append(hyperShiftAPIs(), mce(true)), availableHyperShiftOperator()), wantReady: true, wantReason: "OperandReady"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newPlatformFakeClient(t, tt.objects...) + r := &ClusterInstanceReconciler{Client: c, APIReader: c} + got := r.checkMultiClusterEngine(context.Background()) + if got.ready != tt.wantReady || got.condition.Reason != tt.wantReason { + t.Fatalf("checkMultiClusterEngine() = ready %t, reason %q; want ready %t, reason %q", got.ready, got.condition.Reason, tt.wantReady, tt.wantReason) + } + }) + } +} + +func TestReconcileBlocksHCPBeforeProviderSideEffects(t *testing.T) { + instance := &brokerv1alpha1.ClusterInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "blocked-hcp", Namespace: testNamespace}, + Spec: brokerv1alpha1.ClusterInstanceSpec{Type: brokerv1alpha1.TopologyHCP}, + } + c := newPlatformFakeClient(t, instance, healthyHCO("hco")) + r := &ClusterInstanceReconciler{Client: c, Scheme: c.Scheme(), APIReader: c} + req := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(instance)} + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("adding finalizer: %v", err) + } + result, err := r.Reconcile(context.Background(), req) + if err != nil { + t.Fatalf("blocking reconcile: %v", err) + } + if result.RequeueAfter != requeueInterval { + t.Fatalf("RequeueAfter = %s, want %s", result.RequeueAfter, requeueInterval) + } + + got := &brokerv1alpha1.ClusterInstance{} + if err := c.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("getting ClusterInstance: %v", err) + } + if got.Status.Phase != brokerv1alpha1.PhaseProvisioning { + t.Fatalf("Phase = %q, want %q", got.Status.Phase, brokerv1alpha1.PhaseProvisioning) + } + if condition := apimeta.FindStatusCondition(got.Status.Conditions, conditionTypeMultiClusterEngineReady); condition == nil || condition.Reason != operandNotFoundReason { + t.Fatalf("MultiClusterEngineReady = %+v, want OperandNotFound", condition) + } + namespace := &corev1.Namespace{} + if err := c.Get(context.Background(), client.ObjectKey{Name: "clusters"}, namespace); err == nil { + t.Fatal("HCP reconcile created the clusters namespace while MCE was blocked") + } +} + +func TestGatePlatformOperationsDoesNotRequireMCEForCRC(t *testing.T) { + instance := &brokerv1alpha1.ClusterInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "crc", Namespace: testNamespace}, + Spec: brokerv1alpha1.ClusterInstanceSpec{Type: brokerv1alpha1.TopologyCRC}, + } + c := newPlatformFakeClient(t, instance, healthyHCO("hco")) + r := &ClusterInstanceReconciler{Client: c, APIReader: c} + result, err := r.gatePlatformOperations(context.Background(), instance) + if err != nil || result == nil || result.RequeueAfter != 0 { + t.Fatalf("gatePlatformOperations() = (%+v, %v), want an immediate successful requeue", result, err) + } + if condition := apimeta.FindStatusCondition(instance.Status.Conditions, conditionTypeHyperConvergedReady); condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("HyperConvergedReady = %+v, want True", condition) + } +} + +func TestReconcileReadyHCPProjectsLeaseWhenDependenciesFail(t *testing.T) { + instance := &brokerv1alpha1.ClusterInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "ready-hcp", Namespace: testNamespace, Finalizers: []string{instanceFinalizer}}, + Spec: brokerv1alpha1.ClusterInstanceSpec{Type: brokerv1alpha1.TopologyHCP}, + Status: brokerv1alpha1.ClusterInstanceStatus{Phase: brokerv1alpha1.PhaseReady}, + } + lease := &brokerv1alpha1.ClusterLease{ + ObjectMeta: metav1.ObjectMeta{Name: "lease", Namespace: testNamespace}, + Status: brokerv1alpha1.ClusterLeaseStatus{InstanceRef: &corev1.LocalObjectReference{Name: instance.Name}}, + } + c := newPlatformFakeClient(t, instance, lease) + r := &ClusterInstanceReconciler{Client: c, Scheme: c.Scheme(), APIReader: c} + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(instance)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + got := &brokerv1alpha1.ClusterInstance{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(instance), got); err != nil { + t.Fatalf("getting ClusterInstance: %v", err) + } + if got.Status.LeaseRef == nil || got.Status.LeaseRef.Name != lease.Name { + t.Fatalf("LeaseRef = %+v, want %q", got.Status.LeaseRef, lease.Name) + } +} + +func TestReconcileDeletionBypassesPlatformGate(t *testing.T) { + deletionTime := metav1.NewTime(time.Now()) + instance := &brokerv1alpha1.ClusterInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "deleting-hcp", Namespace: testNamespace, Finalizers: []string{instanceFinalizer}, DeletionTimestamp: &deletionTime}, + Spec: brokerv1alpha1.ClusterInstanceSpec{Type: brokerv1alpha1.TopologyHCP}, + } + c := newPlatformFakeClient(t, instance) + r := &ClusterInstanceReconciler{Client: c, Scheme: c.Scheme(), APIReader: c} + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(instance)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + got := &brokerv1alpha1.ClusterInstance{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(instance), got); err == nil { + t.Fatal("ClusterInstance still exists after deletion") + } +} + +func TestReconcileDoesNotResetFailedInstanceWhenPlatformIsUnavailable(t *testing.T) { + instance := &brokerv1alpha1.ClusterInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "failed", Namespace: testNamespace, Finalizers: []string{instanceFinalizer}}, + Spec: brokerv1alpha1.ClusterInstanceSpec{Type: brokerv1alpha1.TopologyCRC}, + Status: brokerv1alpha1.ClusterInstanceStatus{Phase: brokerv1alpha1.PhaseFailed}, + } + c := newPlatformFakeClient(t, instance) + r := &ClusterInstanceReconciler{Client: c, Scheme: c.Scheme(), APIReader: c} + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(instance)}); err == nil { + t.Fatal("Reconcile succeeded without the required CRC pull secret") + } + got := &brokerv1alpha1.ClusterInstance{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(instance), got); err != nil { + t.Fatalf("getting ClusterInstance: %v", err) + } + if got.Status.Phase != brokerv1alpha1.PhaseFailed { + t.Fatalf("Phase = %q, want %q", got.Status.Phase, brokerv1alpha1.PhaseFailed) + } +} + +func newPlatformFakeClient(t *testing.T, objects ...client.Object) client.Client { + t.Helper() + s := runtime.NewScheme() + if err := scheme.AddToScheme(s); err != nil { + t.Fatalf("adding core scheme: %v", err) + } + if err := appsv1.AddToScheme(s); err != nil { + t.Fatalf("adding apps scheme: %v", err) + } + if err := brokerv1alpha1.AddToScheme(s); err != nil { + t.Fatalf("adding GuestCluster scheme: %v", err) + } + if err := hyperv1beta1.AddToScheme(s); err != nil { + t.Fatalf("adding HyperShift scheme: %v", err) + } + for _, gvk := range []schema.GroupVersionKind{hyperConvergedGVK, multiClusterEngineGVK, customResourceDefinitionGVK} { + s.AddKnownTypeWithName(gvk, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) + } + return fake.NewClientBuilder().WithScheme(s). + WithStatusSubresource(&brokerv1alpha1.ClusterInstance{}, &brokerv1alpha1.ClusterLease{}). + WithIndex(&brokerv1alpha1.ClusterLease{}, leaseInstanceRefIndexField, func(obj client.Object) []string { + lease, ok := obj.(*brokerv1alpha1.ClusterLease) + if !ok || lease.Status.InstanceRef == nil { + return nil + } + return []string{lease.Status.InstanceRef.Name} + }).WithObjects(objects...).Build() +} + +func healthyHCO(name string) *unstructured.Unstructured { + return hco(name, 1, 1, "True", "False", "False") +} + +func hco(name string, generation, observedGeneration int64, available, progressing, degraded string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{ + apiVersionField: hyperConvergedGVK.GroupVersion().String(), kindField: hyperConvergedGVK.Kind, + metadataField: map[string]interface{}{nameField: name, "generation": generation}, + statusField: map[string]interface{}{"observedGeneration": observedGeneration, "conditions": []interface{}{ + map[string]interface{}{conditionTypeField: "Available", statusField: available}, + map[string]interface{}{conditionTypeField: "Progressing", statusField: progressing}, + map[string]interface{}{conditionTypeField: "Degraded", statusField: degraded}, + }}, + }} +} + +func mce(enabled bool) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{ + apiVersionField: multiClusterEngineGVK.GroupVersion().String(), kindField: multiClusterEngineGVK.Kind, + metadataField: map[string]interface{}{nameField: "engine"}, + "spec": map[string]interface{}{"overrides": map[string]interface{}{"components": []interface{}{map[string]interface{}{nameField: "hypershift", "enabled": enabled}}}}, + statusField: map[string]interface{}{"phase": "Progressing"}, + }} +} + +func hyperShiftAPIs() []client.Object { + return []client.Object{establishedCRD("hostedclusters." + hyperShiftGroup), establishedCRD("nodepools." + hyperShiftGroup)} +} + +func establishedCRD(name string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{ + apiVersionField: customResourceDefinitionGVK.GroupVersion().String(), kindField: customResourceDefinitionGVK.Kind, + metadataField: map[string]interface{}{nameField: name}, + statusField: map[string]interface{}{"conditions": []interface{}{map[string]interface{}{conditionTypeField: "Established", statusField: "True"}}}, + }} +} + +func availableHyperShiftOperator() *appsv1.Deployment { + return &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "operator", Namespace: "hypershift"}, Status: appsv1.DeploymentStatus{AvailableReplicas: 1}} +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 3fec5a6..8cd7952 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -96,6 +96,13 @@ var _ = Describe("Manager", Ordered, func() { cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/vmi-crd.yaml") _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to install VirtualMachineInstance CRD") + cmd = exec.Command("kubectl", "wait", "--for=condition=Established", + "crd/hyperconvergeds.hco.kubevirt.io", "--timeout=2m") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to establish HyperConverged CRD") + cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/healthy-hco.yaml") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create healthy HyperConverged") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) @@ -131,6 +138,8 @@ var _ = Describe("Manager", Ordered, func() { _, _ = utils.Run(cmd) By("removing synthetic CRC backing CRDs") + cmd = exec.Command("kubectl", "delete", "-f", "test/e2e/testdata/healthy-hco.yaml", "--ignore-not-found") + _, _ = utils.Run(cmd) cmd = exec.Command("kubectl", "delete", "-f", "test/e2e/testdata/vmi-crd.yaml", "--ignore-not-found") _, _ = utils.Run(cmd) @@ -245,6 +254,7 @@ var _ = Describe("Manager", Ordered, func() { ) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + DeferCleanup(deleteResource, "clusterrolebinding", metricsRoleBindingName) By("validating that the metrics service is available") cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) @@ -439,6 +449,13 @@ spec: "deployment/guestcluster-operator-controller-manager", "-n", namespace, "--timeout=2m") _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "lease", "28494864.opdev.io", "-n", namespace, + "-o", "jsonpath={.spec.holderIdentity}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty()) + }).Should(Succeed()) By("replacing the VMI") cmd = exec.Command("kubectl", "delete", "virtualmachineinstance", instanceName, diff --git a/test/e2e/testdata/healthy-hco.yaml b/test/e2e/testdata/healthy-hco.yaml new file mode 100644 index 0000000..6cbc508 --- /dev/null +++ b/test/e2e/testdata/healthy-hco.yaml @@ -0,0 +1,14 @@ +apiVersion: hco.kubevirt.io/v1beta1 +kind: HyperConverged +metadata: + name: kubevirt-hyperconverged +spec: {} +status: + observedGeneration: 1 + conditions: + - type: Available + status: "True" + - type: Progressing + status: "False" + - type: Degraded + status: "False" diff --git a/test/e2e/testdata/vmi-crd.yaml b/test/e2e/testdata/vmi-crd.yaml index 9c8e353..61c71b2 100644 --- a/test/e2e/testdata/vmi-crd.yaml +++ b/test/e2e/testdata/vmi-crd.yaml @@ -1,5 +1,26 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: hyperconvergeds.hco.kubevirt.io +spec: + group: hco.kubevirt.io + names: + kind: HyperConverged + listKind: HyperConvergedList + plural: hyperconvergeds + singular: hyperconverged + scope: Cluster + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: virtualmachineinstances.kubevirt.io spec: