Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ spec:
- name: config
mountPath: /config
serviceAccountName: controller-manager
terminationGracePeriodSeconds: 10
terminationGracePeriodSeconds: 45
volumes:
- name: config
configMap:
Expand Down
38 changes: 31 additions & 7 deletions internal/cmd/manager/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"fmt"
"os"
"strings"
"time"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
Expand Down Expand Up @@ -230,7 +231,8 @@ func NewCommand(build BuildInfo) *cobra.Command {

setupLog.Info("cluster discovery mode", "mode", serverConfig.Discovery.Mode)

ctx := ctrl.SetupSignalHandler()
shutdown := ctrl.SetupSignalHandler()
ctx, stopManager := context.WithCancel(context.Background())

deploymentClusterClient := deploymentCluster.GetClient()

Expand All @@ -240,10 +242,12 @@ func NewCommand(build BuildInfo) *cobra.Command {
webhook.NewServer(serverConfig.WebhookServer.Options(ctx, deploymentClusterClient)),
serverConfig.Discovery.Mode,
)
go webhookServer.CloseConnectionsThenStop(shutdown, webhookShutdownDelay, stopManager)

leaseDuration := serverConfig.LeaderElection.LeaseDuration.Duration
renewDeadline := serverConfig.LeaderElection.RenewDeadline.Duration
retryPeriod := serverConfig.LeaderElection.RetryPeriod.Duration
gracefulShutdownTimeout := managerGracefulShutdownTimeout

mcManagerOptions := []mcmanager.Option{}
if enableClusterSharding {
Expand Down Expand Up @@ -301,6 +305,7 @@ func NewCommand(build BuildInfo) *cobra.Command {
LeaseDuration: &leaseDuration,
RenewDeadline: &renewDeadline,
RetryPeriod: &retryPeriod,
GracefulShutdownTimeout: &gracefulShutdownTimeout,
LeaderElectionReleaseOnCancel: true,
}, mcManagerOptions...)
if err != nil {
Expand Down Expand Up @@ -349,6 +354,7 @@ func NewCommand(build BuildInfo) *cobra.Command {
LeaseDuration: &leaseDuration,
RenewDeadline: &renewDeadline,
RetryPeriod: &retryPeriod,
GracefulShutdownTimeout: &gracefulShutdownTimeout,
})
if err != nil {
setupLog.Error(err, "unable to create singleton controller manager")
Expand Down Expand Up @@ -377,6 +383,7 @@ func NewCommand(build BuildInfo) *cobra.Command {
LeaseDuration: &leaseDuration,
RenewDeadline: &renewDeadline,
RetryPeriod: &retryPeriod,
GracefulShutdownTimeout: &gracefulShutdownTimeout,
})
if err != nil {
setupLog.Error(err, "unable to create federation controller manager")
Expand Down Expand Up @@ -473,12 +480,8 @@ func NewCommand(build BuildInfo) *cobra.Command {

// +kubebuilder:scaffold:builder

if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
os.Exit(1)
}
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up ready check")
if err := addProbes(mgr); err != nil {
setupLog.Error(err, "unable to set up probes")
os.Exit(1)
}

Expand Down Expand Up @@ -936,6 +939,27 @@ func setupControllers(

const defaultLeaderElectionID = "6a7d51cc.datumapis.com"

const (
webhookShutdownDelay = 10 * time.Second
managerGracefulShutdownTimeout = 30 * time.Second
)

type probeRegistry interface {
AddHealthzCheck(name string, check healthz.Checker) error
AddReadyzCheck(name string, check healthz.Checker) error
GetWebhookServer() webhook.Server
}

func addProbes(mgr probeRegistry) error {
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
return fmt.Errorf("health check: %w", err)
}
if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil {
return fmt.Errorf("ready check: %w", err)
}
return nil
}

func ignoreCanceled(err error) error {
if errors.Is(err, context.Canceled) {
return nil
Expand Down
86 changes: 86 additions & 0 deletions internal/cmd/manager/probes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// SPDX-License-Identifier: AGPL-3.0-only

package managercmd

import (
"context"
"errors"
"net/http"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/webhook"

multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime"

networkingwebhook "go.datum.net/network-services-operator/internal/webhook"
)

type recordedProbes struct {
webhookServer webhook.Server
readyz map[string]healthz.Checker
}

func (r *recordedProbes) AddHealthzCheck(string, healthz.Checker) error { return nil }

func (r *recordedProbes) AddReadyzCheck(name string, check healthz.Checker) error {
r.readyz[name] = check
return nil
}

func (r *recordedProbes) GetWebhookServer() webhook.Server { return r.webhookServer }

func (r *recordedProbes) ready() error {
req, _ := http.NewRequest(http.MethodGet, "/readyz", nil)
var errs []error
for name, check := range r.readyz {
if err := check(req); err != nil {
errs = append(errs, errors.New(name+": "+err.Error()))
}
}
return errors.Join(errs...)
}

func TestManagerIsReadyOnlyWhileTheWebhookServerServes(t *testing.T) {
install := &envtest.WebhookInstallOptions{LocalServingHost: "127.0.0.1"}
require.NoError(t, install.PrepWithoutInstalling())
t.Cleanup(func() { require.NoError(t, install.Cleanup()) })
probes := &recordedProbes{
webhookServer: networkingwebhook.NewClusterAwareWebhookServer(
webhook.NewServer(webhook.Options{
Host: install.LocalServingHost,
Port: install.LocalServingPort,
CertDir: install.LocalServingCertDir,
}),
multiclusterproviders.ProviderMilo,
),
readyz: map[string]healthz.Checker{},
}
if err := addProbes(probes); err != nil {
t.Fatal(err)
}

if err := probes.ready(); err == nil {
t.Fatal("the manager is ready before its webhook server serves")
}

ctx, cancel := context.WithCancel(t.Context())
stopped := make(chan error, 1)
go func() { stopped <- probes.webhookServer.Start(ctx) }()

require.EventuallyWithT(t, func(c *assert.CollectT) {
assert.NoError(c, probes.ready())
}, 10*time.Second, 20*time.Millisecond, "ready once the webhook server serves")

cancel()
if err := <-stopped; err != nil {
t.Fatalf("webhook server stopped with an error: %v", err)
}
if err := probes.ready(); err == nil {
t.Fatal("the manager is ready after its webhook server stopped serving")
}
}
29 changes: 23 additions & 6 deletions internal/webhook/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ package webhook
import (
"context"
"net/http"
"sync/atomic"
"time"

logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
mccontext "sigs.k8s.io/multicluster-runtime/pkg/context"
Expand All @@ -12,14 +15,15 @@ import (
multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime"
)

type clusterAwareWebhookServer struct {
type ClusterAwareWebhookServer struct {
webhook.Server
discoveryMode multiclusterproviders.Provider
closing atomic.Bool
}

var _ webhook.Server = &clusterAwareWebhookServer{}
var _ webhook.Server = &ClusterAwareWebhookServer{}

func (s *clusterAwareWebhookServer) Register(path string, hook http.Handler) {
func (s *ClusterAwareWebhookServer) Register(path string, hook http.Handler) {
if h, ok := hook.(*admission.Webhook); ok {
orig := h.Handler
h.Handler = admission.HandlerFunc(func(ctx context.Context, req admission.Request) admission.Response {
Expand All @@ -29,11 +33,24 @@ func (s *clusterAwareWebhookServer) Register(path string, hook http.Handler) {
})
}

s.Server.Register(path, hook)
s.Server.Register(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if s.closing.Load() {
w.Header().Set("Connection", "close")
}
hook.ServeHTTP(w, r)
}))
}

func NewClusterAwareWebhookServer(server webhook.Server, mode multiclusterproviders.Provider) webhook.Server {
return &clusterAwareWebhookServer{
func (s *ClusterAwareWebhookServer) CloseConnectionsThenStop(shutdown context.Context, delay time.Duration, stopManager context.CancelFunc) {
<-shutdown.Done()
logf.Log.WithName("webhook").Info("closing each connection before shutdown", "delay", delay)
s.closing.Store(true)
time.Sleep(delay)
stopManager()
}

func NewClusterAwareWebhookServer(server webhook.Server, mode multiclusterproviders.Provider) *ClusterAwareWebhookServer {
return &ClusterAwareWebhookServer{
Server: server,
discoveryMode: mode,
}
Expand Down
98 changes: 98 additions & 0 deletions internal/webhook/server_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// SPDX-License-Identifier: AGPL-3.0-only

package webhook_test

import (
"context"
"crypto/tls"
"crypto/x509"
"net"
"net/http"
"strconv"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/webhook"

multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime"

networkingwebhook "go.datum.net/network-services-operator/internal/webhook"
)

func TestServerClosesEachConnectionDuringTheShutdownDelay(t *testing.T) {
const delay = time.Second
install := &envtest.WebhookInstallOptions{LocalServingHost: "127.0.0.1"}
require.NoError(t, install.PrepWithoutInstalling())
t.Cleanup(func() { require.NoError(t, install.Cleanup()) })
server := networkingwebhook.NewClusterAwareWebhookServer(
webhook.NewServer(webhook.Options{
Host: install.LocalServingHost,
Port: install.LocalServingPort,
CertDir: install.LocalServingCertDir,
}),
multiclusterproviders.ProviderMilo,
)
server.Register("/validate", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
url := "https://" + net.JoinHostPort(install.LocalServingHost, strconv.Itoa(install.LocalServingPort)) + "/validate"
client := http1Client(t, install.LocalServingCAData)

shutdown, sigterm := context.WithCancel(t.Context())
managerCtx, stopManager := context.WithCancel(t.Context())
stopped := make(chan error, 1)
go func() { stopped <- server.Start(managerCtx) }()
go server.CloseConnectionsThenStop(shutdown, delay, stopManager)
require.EventuallyWithT(t, func(c *assert.CollectT) {
assert.NoError(c, server.StartedChecker()(nil))
}, 10*time.Second, 20*time.Millisecond, "the webhook server serves")

if resp := post(t, client, url); resp.Close {
t.Fatal("a response before the signal asks the client to close its connection")
}

sigterm()
signalled := time.Now()
require.EventuallyWithT(t, func(c *assert.CollectT) {
assert.True(c, post(c, client, url).Close)
}, 10*time.Second, 20*time.Millisecond, "a response carries Connection: close")
if took := time.Since(signalled); took >= delay {
t.Fatalf("the first response with Connection: close came %v after the signal, not in the delay", took)
}

if err := <-stopped; err != nil {
t.Fatalf("webhook server stopped with an error: %v", err)
}
if took := time.Since(signalled); took < delay {
t.Fatalf("webhook server stopped %v after the signal, before the %v delay", took, delay)
}
if _, err := client.Post(url, "application/json", strings.NewReader("{}")); err == nil {
t.Fatal("webhook server still answers after it stopped")
}
}

func post(t require.TestingT, client *http.Client, url string) *http.Response {
resp, err := client.Post(url, "application/json", strings.NewReader("{}"))
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
return resp
}

// http1Client calls the webhook server over HTTP/1.1 with keep-alive, as the
// API server calls a Service webhook.
func http1Client(t *testing.T, caData []byte) *http.Client {
t.Helper()
roots := x509.NewCertPool()
require.True(t, roots.AppendCertsFromPEM(caData), "the serving CA parses")
return &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: roots, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12},
TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{},
},
}
}
Loading