From b12fbb86a658e907e693a8dac6f53d99bc1b4521 Mon Sep 17 00:00:00 2001 From: Viktor Voytovych Date: Tue, 17 Nov 2026 22:39:02 +0200 Subject: [PATCH 1/2] fix: Close webhook connections before stopping A deleted manager pod stopped its webhook server at once on SIGTERM. The server closed the API server's idle keep-alive connections, and a call the API server sent on one of them at that moment failed with EOF. The manager also reported ready before its webhook server served. Key changes: - on SIGTERM, keep serving for 10 s and answer every webhook call in that time with Connection: close, so each client reconnects through the Service; then stop the manager - set the managers' graceful shutdown timeout to 30 s as a named value - raise terminationGracePeriodSeconds from 10 to 45: the 10 s delay, the 30 s graceful shutdown and 5 s of margin - report ready only while the webhook server accepts TLS connections - share the test webhook server setup in internal/webhook/webhooktest Related to #709 --- config/manager/manager.yaml | 2 +- internal/cmd/manager/manager.go | 38 ++++++++-- internal/cmd/manager/probes_test.go | 77 +++++++++++++++++++++ internal/webhook/server.go | 29 ++++++-- internal/webhook/server_test.go | 73 +++++++++++++++++++ internal/webhook/webhooktest/webhooktest.go | 66 ++++++++++++++++++ 6 files changed, 271 insertions(+), 14 deletions(-) create mode 100644 internal/cmd/manager/probes_test.go create mode 100644 internal/webhook/server_test.go create mode 100644 internal/webhook/webhooktest/webhooktest.go diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d6c75f65..31e940a2 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -104,7 +104,7 @@ spec: - name: config mountPath: /config serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 45 volumes: - name: config configMap: diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 1224b543..afe162d3 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "strings" + "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -230,7 +231,8 @@ func NewCommand(build BuildInfo) *cobra.Command { setupLog.Info("cluster discovery mode", "mode", serverConfig.Discovery.Mode) - ctx := ctrl.SetupSignalHandler() + shutdown := ctrl.SetupSignalHandler() + ctx, stopManager := context.WithCancel(context.Background()) deploymentClusterClient := deploymentCluster.GetClient() @@ -240,10 +242,12 @@ func NewCommand(build BuildInfo) *cobra.Command { webhook.NewServer(serverConfig.WebhookServer.Options(ctx, deploymentClusterClient)), serverConfig.Discovery.Mode, ) + go webhookServer.CloseConnectionsThenStop(shutdown, webhookShutdownDelay, stopManager) leaseDuration := serverConfig.LeaderElection.LeaseDuration.Duration renewDeadline := serverConfig.LeaderElection.RenewDeadline.Duration retryPeriod := serverConfig.LeaderElection.RetryPeriod.Duration + gracefulShutdownTimeout := managerGracefulShutdownTimeout mcManagerOptions := []mcmanager.Option{} if enableClusterSharding { @@ -301,6 +305,7 @@ func NewCommand(build BuildInfo) *cobra.Command { LeaseDuration: &leaseDuration, RenewDeadline: &renewDeadline, RetryPeriod: &retryPeriod, + GracefulShutdownTimeout: &gracefulShutdownTimeout, LeaderElectionReleaseOnCancel: true, }, mcManagerOptions...) if err != nil { @@ -349,6 +354,7 @@ func NewCommand(build BuildInfo) *cobra.Command { LeaseDuration: &leaseDuration, RenewDeadline: &renewDeadline, RetryPeriod: &retryPeriod, + GracefulShutdownTimeout: &gracefulShutdownTimeout, }) if err != nil { setupLog.Error(err, "unable to create singleton controller manager") @@ -377,6 +383,7 @@ func NewCommand(build BuildInfo) *cobra.Command { LeaseDuration: &leaseDuration, RenewDeadline: &renewDeadline, RetryPeriod: &retryPeriod, + GracefulShutdownTimeout: &gracefulShutdownTimeout, }) if err != nil { setupLog.Error(err, "unable to create federation controller manager") @@ -473,12 +480,8 @@ func NewCommand(build BuildInfo) *cobra.Command { // +kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") + if err := addProbes(mgr); err != nil { + setupLog.Error(err, "unable to set up probes") os.Exit(1) } @@ -936,6 +939,27 @@ func setupControllers( const defaultLeaderElectionID = "6a7d51cc.datumapis.com" +const ( + webhookShutdownDelay = 10 * time.Second + managerGracefulShutdownTimeout = 30 * time.Second +) + +type probeRegistry interface { + AddHealthzCheck(name string, check healthz.Checker) error + AddReadyzCheck(name string, check healthz.Checker) error + GetWebhookServer() webhook.Server +} + +func addProbes(mgr probeRegistry) error { + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("health check: %w", err) + } + if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + return fmt.Errorf("ready check: %w", err) + } + return nil +} + func ignoreCanceled(err error) error { if errors.Is(err, context.Canceled) { return nil diff --git a/internal/cmd/manager/probes_test.go b/internal/cmd/manager/probes_test.go new file mode 100644 index 00000000..fabd0cd6 --- /dev/null +++ b/internal/cmd/manager/probes_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package managercmd + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" + + networkingwebhook "go.datum.net/network-services-operator/internal/webhook" + "go.datum.net/network-services-operator/internal/webhook/webhooktest" +) + +type recordedProbes struct { + webhookServer webhook.Server + readyz map[string]healthz.Checker +} + +func (r *recordedProbes) AddHealthzCheck(string, healthz.Checker) error { return nil } + +func (r *recordedProbes) AddReadyzCheck(name string, check healthz.Checker) error { + r.readyz[name] = check + return nil +} + +func (r *recordedProbes) GetWebhookServer() webhook.Server { return r.webhookServer } + +func (r *recordedProbes) ready() error { + req, _ := http.NewRequest(http.MethodGet, "/readyz", nil) + var errs []error + for name, check := range r.readyz { + if err := check(req); err != nil { + errs = append(errs, errors.New(name+": "+err.Error())) + } + } + return errors.Join(errs...) +} + +func TestManagerIsReadyOnlyWhileTheWebhookServerServes(t *testing.T) { + probes := &recordedProbes{ + webhookServer: networkingwebhook.NewClusterAwareWebhookServer( + webhook.NewServer(webhooktest.NewServing(t).Options), + multiclusterproviders.ProviderMilo, + ), + readyz: map[string]healthz.Checker{}, + } + if err := addProbes(probes); err != nil { + t.Fatal(err) + } + + if err := probes.ready(); err == nil { + t.Fatal("the manager is ready before its webhook server serves") + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := make(chan error, 1) + go func() { stopped <- probes.webhookServer.Start(ctx) }() + + webhooktest.WaitFor(t, "ready once the webhook server serves", func(c *assert.CollectT) { + assert.NoError(c, probes.ready()) + }) + + cancel() + if err := <-stopped; err != nil { + t.Fatalf("webhook server stopped with an error: %v", err) + } + if err := probes.ready(); err == nil { + t.Fatal("the manager is ready after its webhook server stopped serving") + } +} diff --git a/internal/webhook/server.go b/internal/webhook/server.go index c22782fd..7712ec6b 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -3,7 +3,10 @@ package webhook import ( "context" "net/http" + "sync/atomic" + "time" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" @@ -12,14 +15,15 @@ import ( multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" ) -type clusterAwareWebhookServer struct { +type ClusterAwareWebhookServer struct { webhook.Server discoveryMode multiclusterproviders.Provider + closing atomic.Bool } -var _ webhook.Server = &clusterAwareWebhookServer{} +var _ webhook.Server = &ClusterAwareWebhookServer{} -func (s *clusterAwareWebhookServer) Register(path string, hook http.Handler) { +func (s *ClusterAwareWebhookServer) Register(path string, hook http.Handler) { if h, ok := hook.(*admission.Webhook); ok { orig := h.Handler h.Handler = admission.HandlerFunc(func(ctx context.Context, req admission.Request) admission.Response { @@ -29,11 +33,24 @@ func (s *clusterAwareWebhookServer) Register(path string, hook http.Handler) { }) } - s.Server.Register(path, hook) + s.Server.Register(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.closing.Load() { + w.Header().Set("Connection", "close") + } + hook.ServeHTTP(w, r) + })) } -func NewClusterAwareWebhookServer(server webhook.Server, mode multiclusterproviders.Provider) webhook.Server { - return &clusterAwareWebhookServer{ +func (s *ClusterAwareWebhookServer) CloseConnectionsThenStop(shutdown context.Context, delay time.Duration, stopManager context.CancelFunc) { + <-shutdown.Done() + logf.Log.WithName("webhook").Info("closing each connection before shutdown", "delay", delay) + s.closing.Store(true) + time.Sleep(delay) + stopManager() +} + +func NewClusterAwareWebhookServer(server webhook.Server, mode multiclusterproviders.Provider) *ClusterAwareWebhookServer { + return &ClusterAwareWebhookServer{ Server: server, discoveryMode: mode, } diff --git a/internal/webhook/server_test.go b/internal/webhook/server_test.go new file mode 100644 index 00000000..8dcf7470 --- /dev/null +++ b/internal/webhook/server_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package webhook_test + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" + + networkingwebhook "go.datum.net/network-services-operator/internal/webhook" + "go.datum.net/network-services-operator/internal/webhook/webhooktest" +) + +func TestServerClosesEachConnectionDuringTheShutdownDelay(t *testing.T) { + const delay = time.Second + serving := webhooktest.NewServing(t) + server := networkingwebhook.NewClusterAwareWebhookServer( + webhook.NewServer(serving.Options), + multiclusterproviders.ProviderMilo, + ) + server.Register("/validate", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + url := serving.URL("/validate") + client := serving.HTTP1Client(t) + + shutdown, sigterm := context.WithCancel(t.Context()) + managerCtx, stopManager := context.WithCancel(t.Context()) + stopped := make(chan error, 1) + go func() { stopped <- server.Start(managerCtx) }() + go server.CloseConnectionsThenStop(shutdown, delay, stopManager) + webhooktest.WaitFor(t, "the webhook server serves", func(c *assert.CollectT) { + assert.NoError(c, server.StartedChecker()(nil)) + }) + + if resp := post(t, client, url); resp.Close { + t.Fatal("a response before the signal asks the client to close its connection") + } + + sigterm() + signalled := time.Now() + webhooktest.WaitFor(t, "a response carries Connection: close", func(c *assert.CollectT) { + assert.True(c, post(c, client, url).Close) + }) + if took := time.Since(signalled); took >= delay { + t.Fatalf("the first response with Connection: close came %v after the signal, not in the delay", took) + } + + if err := <-stopped; err != nil { + t.Fatalf("webhook server stopped with an error: %v", err) + } + if took := time.Since(signalled); took < delay { + t.Fatalf("webhook server stopped %v after the signal, before the %v delay", took, delay) + } + if _, err := client.Post(url, "application/json", strings.NewReader("{}")); err == nil { + t.Fatal("webhook server still answers after it stopped") + } +} + +func post(t require.TestingT, client *http.Client, url string) *http.Response { + resp, err := client.Post(url, "application/json", strings.NewReader("{}")) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + return resp +} diff --git a/internal/webhook/webhooktest/webhooktest.go b/internal/webhook/webhooktest/webhooktest.go new file mode 100644 index 00000000..7dbb158f --- /dev/null +++ b/internal/webhook/webhooktest/webhooktest.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package webhooktest + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +const ( + waitTimeout = 10 * time.Second + pollInterval = 20 * time.Millisecond + clientTimeout = 5 * time.Second + servingHost = "127.0.0.1" +) + +type Serving struct { + Options webhook.Options + caData []byte +} + +func NewServing(t testing.TB) Serving { + t.Helper() + install := &envtest.WebhookInstallOptions{LocalServingHost: servingHost} + require.NoError(t, install.PrepWithoutInstalling()) + t.Cleanup(func() { require.NoError(t, install.Cleanup()) }) + return Serving{ + Options: webhook.Options{ + Host: install.LocalServingHost, + Port: install.LocalServingPort, + CertDir: install.LocalServingCertDir, + }, + caData: install.LocalServingCAData, + } +} + +func (s Serving) URL(path string) string { + return "https://" + s.Options.Host + ":" + strconv.Itoa(s.Options.Port) + path +} + +func (s Serving) HTTP1Client(t testing.TB) *http.Client { + t.Helper() + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(s.caData), "the serving CA parses") + return &http.Client{ + Timeout: clientTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12}, + TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, + }, + } +} + +func WaitFor(t testing.TB, what string, condition func(c *assert.CollectT)) { + t.Helper() + require.EventuallyWithT(t, condition, waitTimeout, pollInterval, "timed out waiting: %s", what) +} From b1e00d9f1a2e734f9e1c762193b2af9cb6ec340d Mon Sep 17 00:00:00 2001 From: Viktor Voytovych Date: Wed, 18 Nov 2026 14:14:56 +0200 Subject: [PATCH 2/2] test: Keep the webhook test helper in the test files The webhooktest package was a non-test package, so `go-licenses report ./...` counted testify in the build and NOTICE had to list it. Each test now sets up its serving certificate in its own _test.go file, which only `go test` compiles, as the finalizer and status tests and controller-runtime's own webhook tests do. --- internal/cmd/manager/probes_test.go | 17 ++++-- internal/webhook/server_test.go | 43 +++++++++++--- internal/webhook/webhooktest/webhooktest.go | 66 --------------------- 3 files changed, 47 insertions(+), 79 deletions(-) delete mode 100644 internal/webhook/webhooktest/webhooktest.go diff --git a/internal/cmd/manager/probes_test.go b/internal/cmd/manager/probes_test.go index fabd0cd6..718610e3 100644 --- a/internal/cmd/manager/probes_test.go +++ b/internal/cmd/manager/probes_test.go @@ -7,15 +7,17 @@ import ( "errors" "net/http" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/webhook" multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" networkingwebhook "go.datum.net/network-services-operator/internal/webhook" - "go.datum.net/network-services-operator/internal/webhook/webhooktest" ) type recordedProbes struct { @@ -44,9 +46,16 @@ func (r *recordedProbes) ready() error { } func TestManagerIsReadyOnlyWhileTheWebhookServerServes(t *testing.T) { + install := &envtest.WebhookInstallOptions{LocalServingHost: "127.0.0.1"} + require.NoError(t, install.PrepWithoutInstalling()) + t.Cleanup(func() { require.NoError(t, install.Cleanup()) }) probes := &recordedProbes{ webhookServer: networkingwebhook.NewClusterAwareWebhookServer( - webhook.NewServer(webhooktest.NewServing(t).Options), + webhook.NewServer(webhook.Options{ + Host: install.LocalServingHost, + Port: install.LocalServingPort, + CertDir: install.LocalServingCertDir, + }), multiclusterproviders.ProviderMilo, ), readyz: map[string]healthz.Checker{}, @@ -63,9 +72,9 @@ func TestManagerIsReadyOnlyWhileTheWebhookServerServes(t *testing.T) { stopped := make(chan error, 1) go func() { stopped <- probes.webhookServer.Start(ctx) }() - webhooktest.WaitFor(t, "ready once the webhook server serves", func(c *assert.CollectT) { + require.EventuallyWithT(t, func(c *assert.CollectT) { assert.NoError(c, probes.ready()) - }) + }, 10*time.Second, 20*time.Millisecond, "ready once the webhook server serves") cancel() if err := <-stopped; err != nil { diff --git a/internal/webhook/server_test.go b/internal/webhook/server_test.go index 8dcf7470..2f5aa369 100644 --- a/internal/webhook/server_test.go +++ b/internal/webhook/server_test.go @@ -4,42 +4,52 @@ package webhook_test import ( "context" + "crypto/tls" + "crypto/x509" + "net" "net/http" + "strconv" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/webhook" multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" networkingwebhook "go.datum.net/network-services-operator/internal/webhook" - "go.datum.net/network-services-operator/internal/webhook/webhooktest" ) func TestServerClosesEachConnectionDuringTheShutdownDelay(t *testing.T) { const delay = time.Second - serving := webhooktest.NewServing(t) + install := &envtest.WebhookInstallOptions{LocalServingHost: "127.0.0.1"} + require.NoError(t, install.PrepWithoutInstalling()) + t.Cleanup(func() { require.NoError(t, install.Cleanup()) }) server := networkingwebhook.NewClusterAwareWebhookServer( - webhook.NewServer(serving.Options), + webhook.NewServer(webhook.Options{ + Host: install.LocalServingHost, + Port: install.LocalServingPort, + CertDir: install.LocalServingCertDir, + }), multiclusterproviders.ProviderMilo, ) server.Register("/validate", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) - url := serving.URL("/validate") - client := serving.HTTP1Client(t) + url := "https://" + net.JoinHostPort(install.LocalServingHost, strconv.Itoa(install.LocalServingPort)) + "/validate" + client := http1Client(t, install.LocalServingCAData) shutdown, sigterm := context.WithCancel(t.Context()) managerCtx, stopManager := context.WithCancel(t.Context()) stopped := make(chan error, 1) go func() { stopped <- server.Start(managerCtx) }() go server.CloseConnectionsThenStop(shutdown, delay, stopManager) - webhooktest.WaitFor(t, "the webhook server serves", func(c *assert.CollectT) { + require.EventuallyWithT(t, func(c *assert.CollectT) { assert.NoError(c, server.StartedChecker()(nil)) - }) + }, 10*time.Second, 20*time.Millisecond, "the webhook server serves") if resp := post(t, client, url); resp.Close { t.Fatal("a response before the signal asks the client to close its connection") @@ -47,9 +57,9 @@ func TestServerClosesEachConnectionDuringTheShutdownDelay(t *testing.T) { sigterm() signalled := time.Now() - webhooktest.WaitFor(t, "a response carries Connection: close", func(c *assert.CollectT) { + require.EventuallyWithT(t, func(c *assert.CollectT) { assert.True(c, post(c, client, url).Close) - }) + }, 10*time.Second, 20*time.Millisecond, "a response carries Connection: close") if took := time.Since(signalled); took >= delay { t.Fatalf("the first response with Connection: close came %v after the signal, not in the delay", took) } @@ -71,3 +81,18 @@ func post(t require.TestingT, client *http.Client, url string) *http.Response { require.NoError(t, resp.Body.Close()) return resp } + +// http1Client calls the webhook server over HTTP/1.1 with keep-alive, as the +// API server calls a Service webhook. +func http1Client(t *testing.T, caData []byte) *http.Client { + t.Helper() + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(caData), "the serving CA parses") + return &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12}, + TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, + }, + } +} diff --git a/internal/webhook/webhooktest/webhooktest.go b/internal/webhook/webhooktest/webhooktest.go deleted file mode 100644 index 7dbb158f..00000000 --- a/internal/webhook/webhooktest/webhooktest.go +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package webhooktest - -import ( - "crypto/tls" - "crypto/x509" - "net/http" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -const ( - waitTimeout = 10 * time.Second - pollInterval = 20 * time.Millisecond - clientTimeout = 5 * time.Second - servingHost = "127.0.0.1" -) - -type Serving struct { - Options webhook.Options - caData []byte -} - -func NewServing(t testing.TB) Serving { - t.Helper() - install := &envtest.WebhookInstallOptions{LocalServingHost: servingHost} - require.NoError(t, install.PrepWithoutInstalling()) - t.Cleanup(func() { require.NoError(t, install.Cleanup()) }) - return Serving{ - Options: webhook.Options{ - Host: install.LocalServingHost, - Port: install.LocalServingPort, - CertDir: install.LocalServingCertDir, - }, - caData: install.LocalServingCAData, - } -} - -func (s Serving) URL(path string) string { - return "https://" + s.Options.Host + ":" + strconv.Itoa(s.Options.Port) + path -} - -func (s Serving) HTTP1Client(t testing.TB) *http.Client { - t.Helper() - roots := x509.NewCertPool() - require.True(t, roots.AppendCertsFromPEM(s.caData), "the serving CA parses") - return &http.Client{ - Timeout: clientTimeout, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: roots, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12}, - TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, - }, - } -} - -func WaitFor(t testing.TB, what string, condition func(c *assert.CollectT)) { - t.Helper() - require.EventuallyWithT(t, condition, waitTimeout, pollInterval, "timed out waiting: %s", what) -}