Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 33 additions & 22 deletions cmd/nonleader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,13 +123,10 @@ func TestSingleReplicaWithoutLeaderElection(t *testing.T) {
defer cancel()
p := startPlanes(t)
downstream := newCluster(t, p.deployment)
zoneReplicator := func(mcmgr mcmanager.Manager) error {
return (&controller.DNSZoneReplicator{DownstreamClient: downstream.GetClient()}).SetupWithManager(mcmgr, downstream)
}
mcmgr := startReplica(ctx, t, p, false, downstream, ctrl.Options{
HealthProbeBindAddress: freeAddress(t),
WebhookServer: neverServing{webhook.NewServer(webhook.Options{})},
}, zoneReplicator, setupReplicatorWebhook)
}, zoneReplicator(downstream), setupReplicatorWebhook)

if !finalizedWithin(ctx, t, newClient(t, p.project), 60*time.Second) {
t.Fatal("the replicator never reconciled the project's DNSZone")
Expand All @@ -140,6 +137,14 @@ func TestSingleReplicaWithoutLeaderElection(t *testing.T) {
}
}

// zoneReplicator sets up the DNSZone replicator, which finalizes each project
// DNSZone it reconciles.
func zoneReplicator(downstream cluster.Cluster) func(mcmanager.Manager) error {
return func(mcmgr mcmanager.Manager) error {
return (&controller.DNSZoneReplicator{DownstreamClient: downstream.GetClient()}).SetupWithManager(mcmgr, downstream)
}
}

func runInOwnProcess(t *testing.T) {
t.Helper()
cmd := exec.Command(os.Args[0], "-test.run=^"+t.Name()+"$", "-test.v")
Expand Down Expand Up @@ -192,7 +197,7 @@ func startReplica(
if contended {
holdLease(ctx, t, deploymentClient)
}
createReadyProject(ctx, t, deploymentClient)
createReadyProject(ctx, t, deploymentClient, testProject)
createZone(ctx, t, newClient(t, p.project))

serverConfig := config.DNSOperator{Discovery: config.DiscoveryConfig{
Expand Down Expand Up @@ -282,21 +287,21 @@ func annotatedWithin(ctx context.Context, t *testing.T, m *dnswebhook.DNSRecordS
}

// goUntilCleanup runs run in the background and, when the test ends, waits for
// it to return. The caller cancels run's context before cleanup. A manager
// still running when envtest stops its API server keeps watches and lease
// renewals open, and the API server then outlasts envtest's stop timeout.
// Cleanups run in reverse order, so every caller after startEnvtest is waited
// for before its API server stops.
// it to return and logs its error. The caller cancels run's context before
// cleanup. A manager still running when envtest stops its API server keeps
// watches and lease renewals open, and the API server then outlasts envtest's
// stop timeout. Cleanups run in reverse order, so every caller after
// startEnvtest is waited for before its API server stops.
func goUntilCleanup(t *testing.T, run func() error) {
t.Helper()
done := make(chan struct{})
go func() {
defer close(done)
_ = run()
}()
done := make(chan error, 1)
go func() { done <- run() }()
t.Cleanup(func() {
select {
case <-done:
case err := <-done:
if err != nil {
t.Logf("a manager stopped with an error: %v", err)
}
case <-time.After(30 * time.Second):
t.Error("a manager did not stop within 30 s of its context ending")
}
Expand Down Expand Up @@ -401,14 +406,11 @@ func holdLease(ctx context.Context, t *testing.T, c client.Client) {
}
}

// createReadyProject creates the test Project in the deployment cluster and
// createReadyProject creates the named Project in the deployment cluster and
// marks it Ready, which is what makes the Milo provider register it.
func createReadyProject(ctx context.Context, t *testing.T, c client.Client) {
func createReadyProject(ctx context.Context, t *testing.T, c client.Client, name string) {
t.Helper()
project := &unstructured.Unstructured{}
project.SetAPIVersion("resourcemanager.miloapis.com/v1alpha1")
project.SetKind("Project")
project.SetName(testProject)
project := projectObject(name)
project.Object["spec"] = map[string]any{"ownerRef": map[string]any{"kind": "Organization", "name": "org"}}
if err := c.Create(ctx, project); err != nil {
t.Fatalf("creating the project: %v", err)
Expand All @@ -422,6 +424,15 @@ func createReadyProject(ctx context.Context, t *testing.T, c client.Client) {
}
}

// projectObject returns the named Milo Project, with no spec or status.
func projectObject(name string) *unstructured.Unstructured {
project := &unstructured.Unstructured{}
project.SetAPIVersion("resourcemanager.miloapis.com/v1alpha1")
project.SetKind("Project")
project.SetName(name)
return project
}

// createZone creates the record set's DNSZone in the project control plane.
func createZone(ctx context.Context, t *testing.T, c client.Client) {
t.Helper()
Expand Down
82 changes: 82 additions & 0 deletions cmd/takeover_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
// SPDX-License-Identifier: AGPL-3.0-only

package main

import (
"context"
"errors"
"os"
"testing"
"time"

coordinationv1 "k8s.io/api/coordination/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
ctrl "sigs.k8s.io/controller-runtime"
mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager"
"sigs.k8s.io/multicluster-runtime/pkg/multicluster"
)

const deletedProject = "p2"

// TestStandbyTakesTheLeaseAfterAProjectIsDeleted requires a standby that takes
// the lease to reconcile its projects after a project it engaged is deleted.
func TestStandbyTakesTheLeaseAfterAProjectIsDeleted(t *testing.T) {
// The Milo provider registers a process-wide controller name, and
// TestNonLeaderReplica starts one, so this test runs in a process of its own.
if os.Getenv(ownProcessEnv) != t.Name() {
runInOwnProcess(t)
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p := startPlanes(t)
deploymentClient := newClient(t, p.deployment)
createReadyProject(ctx, t, deploymentClient, deletedProject)
downstream := newCluster(t, p.deployment)
mcmgr := startReplica(ctx, t, p, true, downstream, ctrl.Options{}, zoneReplicator(downstream))

for _, name := range []string{testProject, deletedProject} {
if err := pollMinute(ctx, engaged(mcmgr, name, true)); err != nil {
t.Fatalf("the standby never engaged project %s: %v", name, err)
}
}
if err := deploymentClient.Delete(ctx, projectObject(deletedProject)); err != nil {
t.Fatalf("deleting project %s: %v", deletedProject, err)
}
if err := pollMinute(ctx, engaged(mcmgr, deletedProject, false)); err != nil {
t.Fatalf("the standby never disengaged the deleted project %s: %v", deletedProject, err)
}

requireNotLeader(t, mcmgr)
lease := &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: replicatorLeaseName, Namespace: testNamespace}}
if err := deploymentClient.Delete(ctx, lease); err != nil {
t.Fatalf("deleting the held lease: %v", err)
}
select {
case <-mcmgr.Elected():
case <-time.After(30 * time.Second):
t.Fatal("the standby did not take the free lease within 30 s")
}
if !finalizedWithin(ctx, t, newClient(t, p.project), 60*time.Second) {
t.Fatal("the new leader never reconciled the DNSZone of the project that remains: " +
"its controller did not start, so the manager stopped")
}
}

// engaged reports whether the replica's provider holds the named project's
// cluster, when want is true, or no longer holds it, when want is false.
func engaged(mcmgr mcmanager.Manager, name string, want bool) wait.ConditionWithContextFunc {
return func(ctx context.Context) (bool, error) {
_, err := mcmgr.GetCluster(ctx, multicluster.ClusterName(name))
if err != nil && !errors.Is(err, multicluster.ErrClusterNotFound) {
return false, err
}
return (err == nil) == want, nil
}
}

// pollMinute checks condition every second until it holds, for a minute.
func pollMinute(ctx context.Context, condition wait.ConditionWithContextFunc) error {
return wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, condition)
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ require (
k8s.io/apimachinery v0.37.2
k8s.io/client-go v0.37.2
sigs.k8s.io/controller-runtime v0.25.2
sigs.k8s.io/multicluster-runtime v0.25.2
sigs.k8s.io/multicluster-runtime v0.25.3-0.20261116153018-6d88a2667465
)

require (
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -479,8 +479,8 @@ sigs.k8s.io/gateway-api v1.5.1 h1:RqVRIlkhLhUO8wOHKTLnTJA6o/1un4po4/6M1nRzdd0=
sigs.k8s.io/gateway-api v1.5.1/go.mod h1:GvCETiaMAlLym5CovLxGjS0NysqFk3+Yuq3/rh6QL2o=
sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5 h1:k9jo7ED/fsfOcz3L1P20P5tjTNtsdQNwMOG2aKzHrl8=
sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/multicluster-runtime v0.25.2 h1:HjcTbBYyrUdLCzjj1zJHFVpEtWEfRuBVYBq3eRToTyc=
sigs.k8s.io/multicluster-runtime v0.25.2/go.mod h1:D/H99r6z2sD9gza69NfePpn81wrwTnCoSmiCK7H8tZ0=
sigs.k8s.io/multicluster-runtime v0.25.3-0.20261116153018-6d88a2667465 h1:yi1bWpgqN9Ubb/UTN7/xSd1FWA1jMEgEfNeTG5MZWuY=
sigs.k8s.io/multicluster-runtime v0.25.3-0.20261116153018-6d88a2667465/go.mod h1:D/H99r6z2sD9gza69NfePpn81wrwTnCoSmiCK7H8tZ0=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q=
Expand Down
Loading