diff --git a/cmd/nonleader_test.go b/cmd/nonleader_test.go index ccc11156..997ea2b7 100644 --- a/cmd/nonleader_test.go +++ b/cmd/nonleader_test.go @@ -123,13 +123,10 @@ func TestSingleReplicaWithoutLeaderElection(t *testing.T) { defer cancel() p := startPlanes(t) downstream := newCluster(t, p.deployment) - zoneReplicator := func(mcmgr mcmanager.Manager) error { - return (&controller.DNSZoneReplicator{DownstreamClient: downstream.GetClient()}).SetupWithManager(mcmgr, downstream) - } mcmgr := startReplica(ctx, t, p, false, downstream, ctrl.Options{ HealthProbeBindAddress: freeAddress(t), WebhookServer: neverServing{webhook.NewServer(webhook.Options{})}, - }, zoneReplicator, setupReplicatorWebhook) + }, zoneReplicator(downstream), setupReplicatorWebhook) if !finalizedWithin(ctx, t, newClient(t, p.project), 60*time.Second) { t.Fatal("the replicator never reconciled the project's DNSZone") @@ -140,6 +137,14 @@ func TestSingleReplicaWithoutLeaderElection(t *testing.T) { } } +// zoneReplicator sets up the DNSZone replicator, which finalizes each project +// DNSZone it reconciles. +func zoneReplicator(downstream cluster.Cluster) func(mcmanager.Manager) error { + return func(mcmgr mcmanager.Manager) error { + return (&controller.DNSZoneReplicator{DownstreamClient: downstream.GetClient()}).SetupWithManager(mcmgr, downstream) + } +} + func runInOwnProcess(t *testing.T) { t.Helper() cmd := exec.Command(os.Args[0], "-test.run=^"+t.Name()+"$", "-test.v") @@ -192,7 +197,7 @@ func startReplica( if contended { holdLease(ctx, t, deploymentClient) } - createReadyProject(ctx, t, deploymentClient) + createReadyProject(ctx, t, deploymentClient, testProject) createZone(ctx, t, newClient(t, p.project)) serverConfig := config.DNSOperator{Discovery: config.DiscoveryConfig{ @@ -282,21 +287,21 @@ func annotatedWithin(ctx context.Context, t *testing.T, m *dnswebhook.DNSRecordS } // goUntilCleanup runs run in the background and, when the test ends, waits for -// it to return. The caller cancels run's context before cleanup. A manager -// still running when envtest stops its API server keeps watches and lease -// renewals open, and the API server then outlasts envtest's stop timeout. -// Cleanups run in reverse order, so every caller after startEnvtest is waited -// for before its API server stops. +// it to return and logs its error. The caller cancels run's context before +// cleanup. A manager still running when envtest stops its API server keeps +// watches and lease renewals open, and the API server then outlasts envtest's +// stop timeout. Cleanups run in reverse order, so every caller after +// startEnvtest is waited for before its API server stops. func goUntilCleanup(t *testing.T, run func() error) { t.Helper() - done := make(chan struct{}) - go func() { - defer close(done) - _ = run() - }() + done := make(chan error, 1) + go func() { done <- run() }() t.Cleanup(func() { select { - case <-done: + case err := <-done: + if err != nil { + t.Logf("a manager stopped with an error: %v", err) + } case <-time.After(30 * time.Second): t.Error("a manager did not stop within 30 s of its context ending") } @@ -401,14 +406,11 @@ func holdLease(ctx context.Context, t *testing.T, c client.Client) { } } -// createReadyProject creates the test Project in the deployment cluster and +// createReadyProject creates the named Project in the deployment cluster and // marks it Ready, which is what makes the Milo provider register it. -func createReadyProject(ctx context.Context, t *testing.T, c client.Client) { +func createReadyProject(ctx context.Context, t *testing.T, c client.Client, name string) { t.Helper() - project := &unstructured.Unstructured{} - project.SetAPIVersion("resourcemanager.miloapis.com/v1alpha1") - project.SetKind("Project") - project.SetName(testProject) + project := projectObject(name) project.Object["spec"] = map[string]any{"ownerRef": map[string]any{"kind": "Organization", "name": "org"}} if err := c.Create(ctx, project); err != nil { t.Fatalf("creating the project: %v", err) @@ -422,6 +424,15 @@ func createReadyProject(ctx context.Context, t *testing.T, c client.Client) { } } +// projectObject returns the named Milo Project, with no spec or status. +func projectObject(name string) *unstructured.Unstructured { + project := &unstructured.Unstructured{} + project.SetAPIVersion("resourcemanager.miloapis.com/v1alpha1") + project.SetKind("Project") + project.SetName(name) + return project +} + // createZone creates the record set's DNSZone in the project control plane. func createZone(ctx context.Context, t *testing.T, c client.Client) { t.Helper() diff --git a/cmd/takeover_test.go b/cmd/takeover_test.go new file mode 100644 index 00000000..c76b82ff --- /dev/null +++ b/cmd/takeover_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + "errors" + "os" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrl "sigs.k8s.io/controller-runtime" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" +) + +const deletedProject = "p2" + +// TestStandbyTakesTheLeaseAfterAProjectIsDeleted requires a standby that takes +// the lease to reconcile its projects after a project it engaged is deleted. +func TestStandbyTakesTheLeaseAfterAProjectIsDeleted(t *testing.T) { + // The Milo provider registers a process-wide controller name, and + // TestNonLeaderReplica starts one, so this test runs in a process of its own. + if os.Getenv(ownProcessEnv) != t.Name() { + runInOwnProcess(t) + return + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p := startPlanes(t) + deploymentClient := newClient(t, p.deployment) + createReadyProject(ctx, t, deploymentClient, deletedProject) + downstream := newCluster(t, p.deployment) + mcmgr := startReplica(ctx, t, p, true, downstream, ctrl.Options{}, zoneReplicator(downstream)) + + for _, name := range []string{testProject, deletedProject} { + if err := pollMinute(ctx, engaged(mcmgr, name, true)); err != nil { + t.Fatalf("the standby never engaged project %s: %v", name, err) + } + } + if err := deploymentClient.Delete(ctx, projectObject(deletedProject)); err != nil { + t.Fatalf("deleting project %s: %v", deletedProject, err) + } + if err := pollMinute(ctx, engaged(mcmgr, deletedProject, false)); err != nil { + t.Fatalf("the standby never disengaged the deleted project %s: %v", deletedProject, err) + } + + requireNotLeader(t, mcmgr) + lease := &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: replicatorLeaseName, Namespace: testNamespace}} + if err := deploymentClient.Delete(ctx, lease); err != nil { + t.Fatalf("deleting the held lease: %v", err) + } + select { + case <-mcmgr.Elected(): + case <-time.After(30 * time.Second): + t.Fatal("the standby did not take the free lease within 30 s") + } + if !finalizedWithin(ctx, t, newClient(t, p.project), 60*time.Second) { + t.Fatal("the new leader never reconciled the DNSZone of the project that remains: " + + "its controller did not start, so the manager stopped") + } +} + +// engaged reports whether the replica's provider holds the named project's +// cluster, when want is true, or no longer holds it, when want is false. +func engaged(mcmgr mcmanager.Manager, name string, want bool) wait.ConditionWithContextFunc { + return func(ctx context.Context) (bool, error) { + _, err := mcmgr.GetCluster(ctx, multicluster.ClusterName(name)) + if err != nil && !errors.Is(err, multicluster.ErrClusterNotFound) { + return false, err + } + return (err == nil) == want, nil + } +} + +// pollMinute checks condition every second until it holds, for a minute. +func pollMinute(ctx context.Context, condition wait.ConditionWithContextFunc) error { + return wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, condition) +} diff --git a/go.mod b/go.mod index 41313de7..b99ca651 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( k8s.io/apimachinery v0.37.2 k8s.io/client-go v0.37.2 sigs.k8s.io/controller-runtime v0.25.2 - sigs.k8s.io/multicluster-runtime v0.25.2 + sigs.k8s.io/multicluster-runtime v0.25.3-0.20261116153018-6d88a2667465 ) require ( diff --git a/go.sum b/go.sum index 6d7d77ed..4f08d655 100644 --- a/go.sum +++ b/go.sum @@ -479,8 +479,8 @@ sigs.k8s.io/gateway-api v1.5.1 h1:RqVRIlkhLhUO8wOHKTLnTJA6o/1un4po4/6M1nRzdd0= sigs.k8s.io/gateway-api v1.5.1/go.mod h1:GvCETiaMAlLym5CovLxGjS0NysqFk3+Yuq3/rh6QL2o= sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5 h1:k9jo7ED/fsfOcz3L1P20P5tjTNtsdQNwMOG2aKzHrl8= sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/multicluster-runtime v0.25.2 h1:HjcTbBYyrUdLCzjj1zJHFVpEtWEfRuBVYBq3eRToTyc= -sigs.k8s.io/multicluster-runtime v0.25.2/go.mod h1:D/H99r6z2sD9gza69NfePpn81wrwTnCoSmiCK7H8tZ0= +sigs.k8s.io/multicluster-runtime v0.25.3-0.20261116153018-6d88a2667465 h1:yi1bWpgqN9Ubb/UTN7/xSd1FWA1jMEgEfNeTG5MZWuY= +sigs.k8s.io/multicluster-runtime v0.25.3-0.20261116153018-6d88a2667465/go.mod h1:D/H99r6z2sD9gza69NfePpn81wrwTnCoSmiCK7H8tZ0= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q=