fix(crc): recover from VMI replacement

README.md

@@ -113,13 +113,14 @@ Two controllers implement this:
     below). The controller also creates a KubeVirt `VirtualMachine` wired
     to that volume. Once the VM's `VirtualMachineInstance` reports an IP,
     the controller creates a run-to-completion **crc-agent Job**
-    (`<instance>-crc-agent`). This Job connects to the VM over SSH as user
+    (`<instance>-crc-agent-<vmi-hash>`). This Job connects to the VM over SSH as user
     `core`, using `template.bundleSSHKeyRef`. It runs every post-boot
     fixup natively, with no external orchestration binary (see
     [crc-agent](#crc-agent-cmdcrc-agent) below). The controller waits for
     the Job to publish the raw kubeconfig Secret
-    (`<instance>-crc-raw-kubeconfig`). Only then does it mark the instance
-    `Ready`. KubeVirt is the only hypervisor involved. No nested
+    (`<instance>-crc-raw-kubeconfig-<vmi-uid>`). It verifies the external API
+    before it marks the instance `Ready`, and retains the result for recovery.
+    KubeVirt is the only hypervisor involved. No nested
     virtualization is required.
   - `hcp`: The controller creates a HyperShift `HostedCluster` with
     `platform: KubeVirt`. It sets `controllerAvailabilityPolicy` from
@@ -414,7 +415,7 @@ For each `ClusterInstance` of topology `crc`, once its
 `VirtualMachineInstance` reports an IP, `ClusterInstanceReconciler`
 ensures a `Service` and passthrough `Route` that expose the guest API
 externally (see below). It then creates a run-to-completion **Kubernetes
-Job** (`<instance>-crc-agent`, see `internal/resources.BuildCRCAgentJob`)
+Job** (`<instance>-crc-agent-<vmi-hash>`, see `internal/resources.BuildCRCAgentJob`)
 that runs this binary. The binary:
 
 1. Waits for the VM's SSH endpoint (port 22) to accept connections. It
@@ -447,14 +448,16 @@ that runs this binary. The binary:
    `https://<CRC_API_HOSTNAME>:443`. It embeds the same self-signed
    certificate as the trusted CA. It derives the cluster's OpenShift
    version from the typed `ClusterVersion` object.
-6. Publishes both values into `<instance>-crc-raw-kubeconfig`, with keys
-   `kubeconfig` and `ocpVersion`. This Secret forms the **only** contract
+6. Publishes both values and the configured VMI UID into
+	`<instance>-crc-raw-kubeconfig-<vmi-uid>`, with keys `kubeconfig`,
+	`ocpVersion`, and `vmiUID`. This Secret forms the **only** contract
    between crc-agent and `ClusterInstanceReconciler`.
    `ClusterInstanceReconciler` reads this Secret and never connects over
    SSH itself.
 
 `BuildCRCAgentJob` sets configuration through environment variables:
 `INSTANCE_NAME`, `INSTANCE_NAMESPACE`, `CRC_SSH_HOST` (the VM's IP),
+`CRC_VMI_UID` (the VMI identity),
 `CRC_API_HOSTNAME` (the externally routable Route host), and
 `CRC_SSH_KEY_PATH` and `PULL_SECRET_PATH` (mounted Secret file paths).
 
@@ -524,8 +527,8 @@ variable, the other a runtime environment variable, read through
 `os.Getenv` by both `ClusterInstanceReconciler` and `CRCBundleReconciler`.
 
 For the crc-agent Job, the image must run as the `crc-agent`
-`ServiceAccount` (`config/rbac/crc_agent_*.yaml`), scoped to `secrets`
-access in the operator's namespace only. For the bundle-prep Job, it must
+`ServiceAccount` (`config/rbac/crc_agent_*.yaml`), scoped to `secrets` and
+its `VirtualMachineInstance` in the operator's namespace. For the bundle-prep Job, it must
 run as the `bundle-prep` `ServiceAccount`, scoped to `secrets` and
 `configmaps` access in the operator's namespace only.
 

api/v1alpha1/clusterinstance_types.go

@@ -77,6 +77,9 @@ type CRCBackingStatus struct {
 	// SSHEndpoint is host:port used by the crc-agent to reach the CRC VM for
 	// post-boot fixups and kubeconfig extraction.
 	SSHEndpoint string `json:"sshEndpoint,omitempty"`
+	// VMIUID identifies the VirtualMachineInstance for which the crc-agent
+	// completed its post-boot handoff.
+	VMIUID string `json:"vmiUID,omitempty"`
 }
 
 // HyperShiftBackingStatus tracks the HostedCluster/NodePool backing a topology=hcp

cmd/crc-agent/cluster.go

@@ -32,6 +32,7 @@ limitations under the License.
 package main
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"encoding/base64"
@@ -47,6 +48,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/caxu-rh/guestcluster-operator/internal/resources"
 )
 
 const (
@@ -104,6 +107,9 @@ type clusterFixupConfig struct {
 	// internal/resources.BuildCRCAPIRoute) used for the external API
 	// serving cert + apiserver namedCertificate patch.
 	APIHostname string
+	// Identity is the management-side certificate material that remains stable
+	// when the backing VMI is replaced.
+	Identity resources.CRCIdentity
 }
 
 // ClusterFixupResult is what a successful RunClusterFixups call returns.
@@ -168,7 +174,9 @@ func RunClusterFixups(
 
 	// 5. External API serving cert + apiserver namedCertificate patch
 	log.Info("cluster: applying external API patches", "apiHostname", cfg.APIHostname)
-	externalAPICACertPEM, err := applyExternalAPIPatches(ctx, clients, cfg.APIHostname)
+	externalAPICACertPEM, err := applyExternalAPIPatches(
+		ctx, clients, cfg.APIHostname, cfg.Identity.ServingCert, cfg.Identity.ServingPrivateKey,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("external API patches: %w", err)
 	}
@@ -326,8 +334,8 @@ func updatePasswords(ctx context.Context, clients *GuestClients, kubeadminPass,
 // External API access (cert + apiserver namedCertificate)
 // ---------------------------------------------------------------------------
 
-// applyExternalAPIPatches generates the self-signed serving certificate for
-// the externally routable API hostname (the management cluster's
+// applyExternalAPIPatches installs the stable serving certificate for the
+// externally routable API hostname (the management cluster's
 // passthrough Route; see internal/resources.BuildCRCAPIRoute). It publishes
 // the certificate as a TLS secret (see externalAPICertSecretName), and it
 // patches the apiserver config so the API server presents that
@@ -343,11 +351,9 @@ func updatePasswords(ctx context.Context, clients *GuestClients, kubeadminPass,
 // not the guest's web console, apps, or image registry. The guest's own
 // VMI network is not routable from outside the management cluster for
 // arbitrary wildcard hostnames the way a single API hostname is.
-func applyExternalAPIPatches(ctx context.Context, clients *GuestClients, apiHostname string) ([]byte, error) {
-	certPEM, keyPEM, err := ExternalAPIServingCert(apiHostname)
-	if err != nil {
-		return nil, fmt.Errorf("generating external API serving cert: %w", err)
-	}
+func applyExternalAPIPatches(
+	ctx context.Context, clients *GuestClients, apiHostname string, certPEM, keyPEM []byte,
+) ([]byte, error) {
 	if err := createOrUpdateTLSSecret(
 		ctx, clients, externalAPICertSecretNamespace, externalAPICertSecretName, certPEM, keyPEM,
 	); err != nil {
@@ -375,8 +381,12 @@ func createOrUpdateTLSSecret(ctx context.Context, clients *GuestClients, ns, nam
 		if getErr != nil {
 			return getErr
 		}
-		existing.Data = sec.Data
-		existing.Type = sec.Type
+		if existing.Type == sec.Type &&
+			bytes.Equal(existing.Data[corev1.TLSCertKey], cert) &&
+			bytes.Equal(existing.Data[corev1.TLSPrivateKeyKey], key) {
+			return nil
+		}
+		existing.Data, existing.Type = sec.Data, sec.Type
 		_, err = clients.Core.CoreV1().Secrets(ns).Update(ctx, existing, metav1.UpdateOptions{})
 	}
 	return err

cmd/crc-agent/guest.go

@@ -24,8 +24,8 @@ limitations under the License.
 //  1. dnsmasq must start so api.crc.testing resolves inside the VM
 //     (needed for `oc` commands issued over SSH).
 //  2. The kubelet must start so the API server comes up.
-//  3. bootstrapCA must regenerate the CA and replace the bundle's stale
-//     admin client cert with a new one. This is the only step that still
+//  3. bootstrapCA installs the stable CA and admin client certificate. This is
+//     the only step that still
 //     runs oc on the guest over SSH. Typed client auth would create a
 //     circular dependency: it needs a valid cert to connect, but it needs
 //     to connect to replace the cert.
@@ -46,8 +46,6 @@ import (
 	"bytes"
 	"crypto/ed25519"
 	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -57,6 +55,8 @@ import (
 
 	gossh "golang.org/x/crypto/ssh"
 	k8syaml "sigs.k8s.io/yaml"
+
+	"github.com/caxu-rh/guestcluster-operator/internal/resources"
 )
 
 // generateEd25519Key generates a fresh ed25519 keypair.
@@ -102,28 +102,19 @@ type guestResult struct {
 	// the new admin client cert injected. The SSH-tunneled typed client uses
 	// this as its TLS credential.
 	AdminKubeconfigPEM []byte
-	// CACert is the self-signed CA that bootstrapCA generates. The API server
-	// trusts this CA for verifying client certificates through the
-	// admin-kubeconfig-client-ca configmap (it signs ClientCertPEM below).
-	// It does not sign the API server's own TLS serving certificate. See
-	// ServerCAPEM for that.
-	CACert *x509.Certificate
-	// CAKey is the matching private key, kept for later use if needed.
-	CAKey *rsa.PrivateKey
-	// ClientCertPEM and ClientKeyPEM are the admin client cert and key PEM
-	// bytes, signed by CACert.
+	// ClientCertPEM and ClientKeyPEM are the mounted admin client credential.
 	ClientCertPEM []byte
 	ClientKeyPEM  []byte
 	// ServerCAPEM is the CA bundle that verifies the API server's own TLS
 	// serving certificate for the internal api.crc.testing SNI. bootstrapCA
 	// extracts it from the bundle kubeconfig's
 	// clusters[0].cluster.certificate-authority-data field, which it never
-	// modifies. This is a different trust root than CACert: it is whatever
+	// modifies. This is separate from the mounted client CA: it is whatever
 	// the CRC bundle was built with (for example kube-apiserver-lb-signer
 	// and related certs), not something crc-agent generates. Typed clients
 	// connecting to api.crc.testing (clusterclient.go) must use this as
-	// their CAData, not CACert. Conflating the two produces "certificate
-	// signed by unknown authority", because CACert never signed the
+	// their CAData, not the client CA. Conflating the two produces "certificate
+	// signed by unknown authority", because the client CA never signed the
 	// server's serving cert.
 	ServerCAPEM []byte
 }
@@ -154,7 +145,7 @@ nameserver {{ .IP }}
 //  2. Rewrite /etc/resolv.conf so the VM resolves *.crc.testing.
 //  3. Generate a new ed25519 SSH keypair and swap it onto authorized_keys.
 //  4. Start the kubelet.
-//  5. Regenerate the admin CA and client cert, then patch the cluster
+//  5. Install the stable admin CA and client cert, then patch the cluster
 //     (the CA bootstrap step; this uses oc on the guest for this one step).
 //  6. Return the new kubeconfig bytes and crypto material for the
 //     typed-client stage.
@@ -196,9 +187,9 @@ func RunGuestFixups(runner *Runner, cfg config, log logrLike) (*guestResult, err
 		return nil, fmt.Errorf("start kubelet: %w", err)
 	}
 
-	// 5. CA bootstrap: generate new CA+client cert, patch the cluster
-	log.Info("guest: regenerating admin CA and client cert (bootstrap)")
-	res, err := bootstrapCA(runner)
+	// 5. CA bootstrap: install the stable CA+client cert and patch the cluster.
+	log.Info("guest: installing stable admin CA and client cert")
+	res, err := bootstrapCA(runner, cfg.Identity)
 	if err != nil {
 		return nil, fmt.Errorf("bootstrap CA: %w", err)
 	}
@@ -340,10 +331,8 @@ func startKubelet(runner *Runner) error {
 	return nil
 }
 
-// bootstrapCA is the CA-regen bootstrap step:
-//  1. Generate a self-signed CA (crypto/x509).
-//  2. Mint a system:admin / system:masters client cert.
-//  3. Read the bundle's /opt/kubeconfig and splice in the new client cert.
+// bootstrapCA installs the management-side identity into the guest:
+//  1. Read the bundle's /opt/kubeconfig and splice in the stable client cert.
 //  4. Run `oc patch configmap admin-kubeconfig-client-ca` on the guest over
 //     SSH so the API server trusts the new CA. This is the one intentional
 //     oc-on-guest exception, matching crc's own approach: the API server
@@ -352,17 +341,7 @@ func startKubelet(runner *Runner) error {
 //
 // Ported from crc pkg/crc/machine/start.go updateKubeconfig and
 // pkg/crc/cluster/cluster.go EnsureGeneratedClientCAPresentInTheCluster.
-func bootstrapCA(runner *Runner) (*guestResult, error) {
-	// Generate CA + client cert.
-	caKey, caCert, err := SelfSignedCA()
-	if err != nil {
-		return nil, err
-	}
-	clientCertPEM, clientKeyPEM, err := ClientCertificate(caKey, caCert)
-	if err != nil {
-		return nil, err
-	}
-	caPEM := CAPem(caCert)
+func bootstrapCA(runner *Runner, identity resources.CRCIdentity) (*guestResult, error) {
 
 	// Read the bundle's admin kubeconfig from the guest, with retries. The
 	// caller started the kubelet a moment ago, and the kubelet may not have
@@ -382,15 +361,15 @@ func bootstrapCA(runner *Runner) (*guestResult, error) {
 
 	// Splice in the new client cert+key.
 	patchedKubeconfig, err := spliceClientCertIntoKubeconfig(
-		[]byte(bundleKubeconfigYAML), clientCertPEM, clientKeyPEM)
+		[]byte(bundleKubeconfigYAML), identity.ClientCert, identity.ClientPrivateKey)
 	if err != nil {
 		return nil, fmt.Errorf("splicing client cert into kubeconfig: %w", err)
 	}
 
 	// Patch admin-kubeconfig-client-ca configmap via oc on the guest.
 	// This is the only oc-on-guest call; it must happen BEFORE typed clients
 	// try to authenticate, so the API server trusts our new CA.
-	caPEMJSON, _ := json.Marshal(string(caPEM))
+	caPEMJSON, _ := json.Marshal(string(identity.ClientCA))
 	patchCmd := fmt.Sprintf(
 		`oc --kubeconfig /opt/kubeconfig patch configmap admin-kubeconfig-client-ca `+
 			`-n openshift-config --patch '{"data":{"ca-bundle.crt":%s}}'`,
@@ -429,11 +408,9 @@ func bootstrapCA(runner *Runner) (*guestResult, error) {
 
 	return &guestResult{
 		AdminKubeconfigPEM: patchedKubeconfig,
-		CACert:             caCert,
-		CAKey:              caKey,
 		ServerCAPEM:        serverCAPEM,
-		ClientCertPEM:      clientCertPEM,
-		ClientKeyPEM:       clientKeyPEM,
+		ClientCertPEM:      identity.ClientCert,
+		ClientKeyPEM:       identity.ClientPrivateKey,
 	}, nil
 }