diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d6c75f65..31e940a2 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -104,7 +104,7 @@ spec: - name: config mountPath: /config serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 45 volumes: - name: config configMap: diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 1224b543..afe162d3 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "strings" + "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -230,7 +231,8 @@ func NewCommand(build BuildInfo) *cobra.Command { setupLog.Info("cluster discovery mode", "mode", serverConfig.Discovery.Mode) - ctx := ctrl.SetupSignalHandler() + shutdown := ctrl.SetupSignalHandler() + ctx, stopManager := context.WithCancel(context.Background()) deploymentClusterClient := deploymentCluster.GetClient() @@ -240,10 +242,12 @@ func NewCommand(build BuildInfo) *cobra.Command { webhook.NewServer(serverConfig.WebhookServer.Options(ctx, deploymentClusterClient)), serverConfig.Discovery.Mode, ) + go webhookServer.CloseConnectionsThenStop(shutdown, webhookShutdownDelay, stopManager) leaseDuration := serverConfig.LeaderElection.LeaseDuration.Duration renewDeadline := serverConfig.LeaderElection.RenewDeadline.Duration retryPeriod := serverConfig.LeaderElection.RetryPeriod.Duration + gracefulShutdownTimeout := managerGracefulShutdownTimeout mcManagerOptions := []mcmanager.Option{} if enableClusterSharding { @@ -301,6 +305,7 @@ func NewCommand(build BuildInfo) *cobra.Command { LeaseDuration: &leaseDuration, RenewDeadline: &renewDeadline, RetryPeriod: &retryPeriod, + GracefulShutdownTimeout: &gracefulShutdownTimeout, LeaderElectionReleaseOnCancel: true, }, mcManagerOptions...) if err != nil { @@ -349,6 +354,7 @@ func NewCommand(build BuildInfo) *cobra.Command { LeaseDuration: &leaseDuration, RenewDeadline: &renewDeadline, RetryPeriod: &retryPeriod, + GracefulShutdownTimeout: &gracefulShutdownTimeout, }) if err != nil { setupLog.Error(err, "unable to create singleton controller manager") @@ -377,6 +383,7 @@ func NewCommand(build BuildInfo) *cobra.Command { LeaseDuration: &leaseDuration, RenewDeadline: &renewDeadline, RetryPeriod: &retryPeriod, + GracefulShutdownTimeout: &gracefulShutdownTimeout, }) if err != nil { setupLog.Error(err, "unable to create federation controller manager") @@ -473,12 +480,8 @@ func NewCommand(build BuildInfo) *cobra.Command { // +kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") + if err := addProbes(mgr); err != nil { + setupLog.Error(err, "unable to set up probes") os.Exit(1) } @@ -936,6 +939,27 @@ func setupControllers( const defaultLeaderElectionID = "6a7d51cc.datumapis.com" +const ( + webhookShutdownDelay = 10 * time.Second + managerGracefulShutdownTimeout = 30 * time.Second +) + +type probeRegistry interface { + AddHealthzCheck(name string, check healthz.Checker) error + AddReadyzCheck(name string, check healthz.Checker) error + GetWebhookServer() webhook.Server +} + +func addProbes(mgr probeRegistry) error { + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("health check: %w", err) + } + if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + return fmt.Errorf("ready check: %w", err) + } + return nil +} + func ignoreCanceled(err error) error { if errors.Is(err, context.Canceled) { return nil diff --git a/internal/cmd/manager/probes_test.go b/internal/cmd/manager/probes_test.go new file mode 100644 index 00000000..718610e3 --- /dev/null +++ b/internal/cmd/manager/probes_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package managercmd + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" + + networkingwebhook "go.datum.net/network-services-operator/internal/webhook" +) + +type recordedProbes struct { + webhookServer webhook.Server + readyz map[string]healthz.Checker +} + +func (r *recordedProbes) AddHealthzCheck(string, healthz.Checker) error { return nil } + +func (r *recordedProbes) AddReadyzCheck(name string, check healthz.Checker) error { + r.readyz[name] = check + return nil +} + +func (r *recordedProbes) GetWebhookServer() webhook.Server { return r.webhookServer } + +func (r *recordedProbes) ready() error { + req, _ := http.NewRequest(http.MethodGet, "/readyz", nil) + var errs []error + for name, check := range r.readyz { + if err := check(req); err != nil { + errs = append(errs, errors.New(name+": "+err.Error())) + } + } + return errors.Join(errs...) +} + +func TestManagerIsReadyOnlyWhileTheWebhookServerServes(t *testing.T) { + install := &envtest.WebhookInstallOptions{LocalServingHost: "127.0.0.1"} + require.NoError(t, install.PrepWithoutInstalling()) + t.Cleanup(func() { require.NoError(t, install.Cleanup()) }) + probes := &recordedProbes{ + webhookServer: networkingwebhook.NewClusterAwareWebhookServer( + webhook.NewServer(webhook.Options{ + Host: install.LocalServingHost, + Port: install.LocalServingPort, + CertDir: install.LocalServingCertDir, + }), + multiclusterproviders.ProviderMilo, + ), + readyz: map[string]healthz.Checker{}, + } + if err := addProbes(probes); err != nil { + t.Fatal(err) + } + + if err := probes.ready(); err == nil { + t.Fatal("the manager is ready before its webhook server serves") + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := make(chan error, 1) + go func() { stopped <- probes.webhookServer.Start(ctx) }() + + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.NoError(c, probes.ready()) + }, 10*time.Second, 20*time.Millisecond, "ready once the webhook server serves") + + cancel() + if err := <-stopped; err != nil { + t.Fatalf("webhook server stopped with an error: %v", err) + } + if err := probes.ready(); err == nil { + t.Fatal("the manager is ready after its webhook server stopped serving") + } +} diff --git a/internal/webhook/server.go b/internal/webhook/server.go index c22782fd..7712ec6b 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -3,7 +3,10 @@ package webhook import ( "context" "net/http" + "sync/atomic" + "time" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" @@ -12,14 +15,15 @@ import ( multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" ) -type clusterAwareWebhookServer struct { +type ClusterAwareWebhookServer struct { webhook.Server discoveryMode multiclusterproviders.Provider + closing atomic.Bool } -var _ webhook.Server = &clusterAwareWebhookServer{} +var _ webhook.Server = &ClusterAwareWebhookServer{} -func (s *clusterAwareWebhookServer) Register(path string, hook http.Handler) { +func (s *ClusterAwareWebhookServer) Register(path string, hook http.Handler) { if h, ok := hook.(*admission.Webhook); ok { orig := h.Handler h.Handler = admission.HandlerFunc(func(ctx context.Context, req admission.Request) admission.Response { @@ -29,11 +33,24 @@ func (s *clusterAwareWebhookServer) Register(path string, hook http.Handler) { }) } - s.Server.Register(path, hook) + s.Server.Register(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.closing.Load() { + w.Header().Set("Connection", "close") + } + hook.ServeHTTP(w, r) + })) } -func NewClusterAwareWebhookServer(server webhook.Server, mode multiclusterproviders.Provider) webhook.Server { - return &clusterAwareWebhookServer{ +func (s *ClusterAwareWebhookServer) CloseConnectionsThenStop(shutdown context.Context, delay time.Duration, stopManager context.CancelFunc) { + <-shutdown.Done() + logf.Log.WithName("webhook").Info("closing each connection before shutdown", "delay", delay) + s.closing.Store(true) + time.Sleep(delay) + stopManager() +} + +func NewClusterAwareWebhookServer(server webhook.Server, mode multiclusterproviders.Provider) *ClusterAwareWebhookServer { + return &ClusterAwareWebhookServer{ Server: server, discoveryMode: mode, } diff --git a/internal/webhook/server_test.go b/internal/webhook/server_test.go new file mode 100644 index 00000000..2f5aa369 --- /dev/null +++ b/internal/webhook/server_test.go @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package webhook_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" + + networkingwebhook "go.datum.net/network-services-operator/internal/webhook" +) + +func TestServerClosesEachConnectionDuringTheShutdownDelay(t *testing.T) { + const delay = time.Second + install := &envtest.WebhookInstallOptions{LocalServingHost: "127.0.0.1"} + require.NoError(t, install.PrepWithoutInstalling()) + t.Cleanup(func() { require.NoError(t, install.Cleanup()) }) + server := networkingwebhook.NewClusterAwareWebhookServer( + webhook.NewServer(webhook.Options{ + Host: install.LocalServingHost, + Port: install.LocalServingPort, + CertDir: install.LocalServingCertDir, + }), + multiclusterproviders.ProviderMilo, + ) + server.Register("/validate", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + url := "https://" + net.JoinHostPort(install.LocalServingHost, strconv.Itoa(install.LocalServingPort)) + "/validate" + client := http1Client(t, install.LocalServingCAData) + + shutdown, sigterm := context.WithCancel(t.Context()) + managerCtx, stopManager := context.WithCancel(t.Context()) + stopped := make(chan error, 1) + go func() { stopped <- server.Start(managerCtx) }() + go server.CloseConnectionsThenStop(shutdown, delay, stopManager) + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.NoError(c, server.StartedChecker()(nil)) + }, 10*time.Second, 20*time.Millisecond, "the webhook server serves") + + if resp := post(t, client, url); resp.Close { + t.Fatal("a response before the signal asks the client to close its connection") + } + + sigterm() + signalled := time.Now() + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, post(c, client, url).Close) + }, 10*time.Second, 20*time.Millisecond, "a response carries Connection: close") + if took := time.Since(signalled); took >= delay { + t.Fatalf("the first response with Connection: close came %v after the signal, not in the delay", took) + } + + if err := <-stopped; err != nil { + t.Fatalf("webhook server stopped with an error: %v", err) + } + if took := time.Since(signalled); took < delay { + t.Fatalf("webhook server stopped %v after the signal, before the %v delay", took, delay) + } + if _, err := client.Post(url, "application/json", strings.NewReader("{}")); err == nil { + t.Fatal("webhook server still answers after it stopped") + } +} + +func post(t require.TestingT, client *http.Client, url string) *http.Response { + resp, err := client.Post(url, "application/json", strings.NewReader("{}")) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + return resp +} + +// http1Client calls the webhook server over HTTP/1.1 with keep-alive, as the +// API server calls a Service webhook. +func http1Client(t *testing.T, caData []byte) *http.Client { + t.Helper() + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(caData), "the serving CA parses") + return &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12}, + TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, + }, + } +}