test: add the multi-controller suite, with the three fixes it proves

.github/workflows/build-and-release.yaml

@@ -50,7 +50,9 @@ jobs:
           cache: false
 
       - name: Run tests
-        run: go test ./... -coverprofile=./cover.out -covermode=atomic -coverpkg=./...
+        run: |
+          go test $(go list ./... | grep -v /test/suite) \
+            -coverprofile=./cover.out -covermode=atomic -coverpkg=./...
 
       - name: Run observer tests against current operator API
         working-directory: tools/observer

.github/workflows/test-suite.yaml

@@ -0,0 +1,40 @@
+# This job is advisory. It is deliberately not a required check and is not
+# referenced by any other workflow's needs. A red run here means the suite
+# caught a real problem, not that CI itself is broken: treat a failure as a
+# finding to investigate, not as noise to wave off or restart until it goes
+# green.
+name: Test suite
+
+on:
+  pull_request: {}
+  # main only: pull_request already covers any branch with a PR open, so a
+  # wider push trigger would run the suite twice for it. main is here to catch
+  # a bad merge.
+  push:
+    branches:
+      - main
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  test-suite:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - name: Check out code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Install Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Run test suite
+        run: make test-suite

.golangci.toml

@@ -10,11 +10,16 @@ enable = [ "gosec" ]
 # for a dedicated follow-up migration; excluded here so the dependency bump
 # that introduced the deprecations isn't blocked on an unrelated,
 # wide-reaching refactor.
+#
+# The optional package-path prefix is there because staticcheck names the
+# symbol differently across versions: bare ("client.Apply") before
+# golangci-lint v2.13, fully qualified ("sigs.k8s.io/.../pkg/client.Apply")
+# from it.
 rules = [
-  { linters = [ "staticcheck" ], text = "SA1019: client.Apply is deprecated" },
-  { linters = [ "staticcheck" ], text = "SA1019: scheme.Builder is deprecated" },
-  { linters = [ "staticcheck" ], text = "SA1019: webhook.CustomDefaulter is deprecated" },
-  { linters = [ "staticcheck" ], text = "SA1019: webhook.CustomValidator is deprecated" },
+  { linters = [ "staticcheck" ], text = 'SA1019: (\S+/)?client\.Apply is deprecated' },
+  { linters = [ "staticcheck" ], text = 'SA1019: (\S+/)?scheme\.Builder is deprecated' },
+  { linters = [ "staticcheck" ], text = 'SA1019: (\S+/)?webhook\.CustomDefaulter is deprecated' },
+  { linters = [ "staticcheck" ], text = 'SA1019: (\S+/)?webhook\.CustomValidator is deprecated' },
   { linters = [ "staticcheck" ], text = "WithCustomDefaulter is deprecated" },
   { linters = [ "staticcheck" ], text = "WithCustomValidator is deprecated" },
   { linters = [ "staticcheck" ], text = "GetEventRecorderFor is deprecated" },

Dockerfile

@@ -1,6 +1,6 @@
 # Containerfile for multigres-operator
 
-FROM --platform=$BUILDPLATFORM golang:1.26.6-alpine3.23 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.27.1-alpine3.23 AS builder
 
 ARG TARGETOS
 ARG TARGETARCH

Makefile

@@ -108,7 +108,7 @@ KUSTOMIZE_VERSION ?= v5.6.0
 # renovate: datasource=github-releases depName=kubernetes-sigs/controller-tools
 CONTROLLER_TOOLS_VERSION ?= v0.18.0
 # renovate: datasource=github-releases depName=golangci/golangci-lint
-GOLANGCI_LINT_VERSION ?= v2.12.2
+GOLANGCI_LINT_VERSION ?= v2.13.2
 
 CERT_MANAGER_VERSION ?= v1.19.2
 
@@ -278,22 +278,65 @@ build-installer: manifests generate kustomize ## Generate consolidated install Y
 
 ##@ Test
 
+# test/suite is the multi-controller envtest suite. It carries no build tag, so
+# every `go test ./...` call site has to exclude it by path or it lands in the
+# required check before it is ready. That is one filter per call site, which is
+# the deliberate trade against a tag that someone forgets on a new file.
+#
+# -v is load-bearing rather than cosmetic. Each KnownDefect pin logs the defect
+# it is standing on while that defect is still present, and without -v go test
+# discards the output of a passing test, so a green CI run shows none of them.
+# The suite is meant to be readable as the operator's live defect list, and -v
+# is what makes that list visible without waiting for a pin to expire.
+.PHONY: test-suite
+test-suite: manifests generate fmt vet setup-envtest ## Run the multi-controller test suite
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
+		go test -v -p 1 -timeout 20m ./test/suite/...
+
+# A separate target rather than a flag on the one above. Measured 2026-09-19:
+# 183s against a 166s baseline, so about 10% rather than the roughly-double a
+# CPU-bound suite would pay. This one spends most of its wall clock waiting for
+# controllers to converge, and the race detector does not slow down waiting.
+#
+# Kept separate anyway, because the cost is not the same everywhere: certificate
+# generation is the one CPU-bound step here and has been measured swinging
+# between 14 and 75 seconds under -race, which is enough to turn a wait sized
+# against the normal run into a flake. A budget that holds on both is looser
+# than the default target should carry.
+#
+# Worth having at all because this suite is the only place five controllers
+# share one manager, and the operator holds exactly one piece of state across
+# reconcile goroutines: ShardReconciler.postureStrikes, a map guarded by a
+# mutex. Nothing here exercises contention on it today, since the suite pins
+# MaxConcurrentReconciles to 1 and controller-runtime already serialises
+# reconciles per object key, so this is a standing check that the answer has
+# not changed rather than a hunt for a known race.
+#
+# The timeout is generous rather than tight: the instrumented run is only
+# slightly slower on average, but its slow tail is much fatter, and a timeout
+# that fires on the tail reads as a hang rather than as the flake it is.
+.PHONY: test-suite-race
+test-suite-race: manifests generate fmt vet setup-envtest ## Run the multi-controller test suite under the race detector
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
+		go test -race -v -p 1 -timeout 40m ./test/suite/...
+
 .PHONY: test
 test: manifests generate fmt vet ## Run tests (no integration testing)
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
-		go test -p 1 $$(go list ./... | grep -v /e2e) -coverprofile=cover.out
+		go test -p 1 $$(go list ./... | grep -v /e2e | grep -v /test/suite) -coverprofile=cover.out
 
 .PHONY: test-integration
 test-integration: manifests generate fmt vet setup-envtest ## Run integration tests
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
-		go test -p 1 -tags=integration,verbose $$(go list ./... | grep -v /e2e) -coverprofile=cover.out
+		go test -p 1 -tags=integration,verbose $$(go list ./... | grep -v /e2e | grep -v /test/suite) -coverprofile=cover.out
 
 .PHONY: test-coverage
 test-coverage: manifests generate fmt vet setup-envtest ## Generate coverage report with HTML
 	@mkdir -p coverage
 	@echo "==> Generating coverage..."
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
-		go test -p 1 -tags=integration,verbose ./... -coverprofile=coverage/combined.out -covermode=atomic
+		go test -p 1 -tags=integration,verbose $$(go list ./... | grep -v /e2e | grep -v /test/suite) \
+		-coverprofile=coverage/combined.out -covermode=atomic
 	@echo "==> Generating HTML report..."
 	@go tool cover -html=coverage/combined.out -o=coverage/combined.html
 	@echo "Generated: coverage/combined.html"
@@ -679,10 +722,13 @@ $(ENVTEST): $(LOCALBIN)
 golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 # golangci-lint's own go.mod selects an older toolchain than this module
 # targets, and a linter built with a lower Go version refuses to run. Pin the
-# build toolchain to the one resolved by this module's go.mod.
+# build toolchain to the one resolved by this module's go.mod, and put that
+# version in the binary's name: CI restores bin/ from older caches, and a
+# name keyed only on the linter's version would reuse a binary built by the
+# previous toolchain after a Go bump.
 $(GOLANGCI_LINT): export GOTOOLCHAIN = $(shell go env GOVERSION)
 $(GOLANGCI_LINT): $(LOCALBIN)
-	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))
+	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION),$(shell go env GOVERSION))
 
 .PHONY: install-certmanager
 install-certmanager: ## Install Cert-Manager into the cluster
@@ -695,16 +741,17 @@ install-certmanager: ## Install Cert-Manager into the cluster
 # $1 - target path with name of binary
 # $2 - package url which can be installed
 # $3 - specific version of package
+# $4 - optional extra suffix for the binary's name, e.g. the Go version it was built with
 define go-install-tool
-@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \
+@[ -f "$(1)-$(3)$(if $(4),-$(4))" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)$(if $(4),-$(4))" ] || { \
 set -e; \
 package=$(2)@$(3) ;\
 echo "Downloading $${package}" ;\
 rm -f $(1) ;\
 GOBIN=$(LOCALBIN) go install $${package} ;\
-mv $(1) $(1)-$(3) ;\
+mv $(1) $(1)-$(3)$(if $(4),-$(4)) ;\
 } ;\
-ln -sf $$(realpath $(1)-$(3)) $(1)
+ln -sf $$(realpath $(1)-$(3)$(if $(4),-$(4))) $(1)
 endef
 
 ##@ Backward Compatibility Aliases

go.mod

@@ -1,11 +1,12 @@
 module github.com/multigres/multigres-operator
 
-go 1.26.6
+go 1.27.1
 
 require (
 	github.com/go-logr/logr v1.4.4
 	github.com/google/go-cmp v0.7.0
 	github.com/multigres/multigres v0.0.0-20260925193740-522b90425a83
+	github.com/multigres/testkit v0.2.1
 	github.com/prometheus/client_golang v1.24.1
 	github.com/prometheus/client_model v0.6.3
 	github.com/stretchr/testify v1.12.1
@@ -15,14 +16,16 @@ require (
 	go.opentelemetry.io/otel v1.46.0
 	go.opentelemetry.io/otel/sdk v1.46.0
 	go.opentelemetry.io/otel/trace v1.46.0
+	go.uber.org/goleak v1.3.0
 	google.golang.org/grpc v1.83.2
 	google.golang.org/protobuf v1.36.12
 	k8s.io/api v0.37.0
 	k8s.io/apimachinery v0.37.0
 	k8s.io/client-go v0.37.0
-	k8s.io/utils v0.0.0-20260626114624-be93311217bd
-	sigs.k8s.io/controller-runtime v0.25.0
+	k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3
+	sigs.k8s.io/controller-runtime v0.25.1
 	sigs.k8s.io/e2e-framework v0.7.0
+	sigs.k8s.io/structured-merge-diff/v6 v6.4.2
 )
 
 require (
@@ -128,7 +131,6 @@ require (
 	go.opentelemetry.io/otel/sdk/log v0.22.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
-	go.uber.org/goleak v1.3.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
@@ -157,7 +159,6 @@ require (
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
-	sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect
 )
 

go.sum

@@ -199,6 +199,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/multigres/multigres v0.0.0-20260925193740-522b90425a83 h1:IdyFGtwc9pEZEzqs6h5JfavAnErWfuKwj3d42qkZUx8=
 github.com/multigres/multigres v0.0.0-20260925193740-522b90425a83/go.mod h1:Ov2hrkOguWSkCS2QIhAdguFeG5GlZ3v4WGqIdqkQ7Tg=
+github.com/multigres/testkit v0.2.1 h1:1SOV2jevblZBpobzaoFAy/lVqb2Donihjc+EovmbIAo=
+github.com/multigres/testkit v0.2.1/go.mod h1:3ONhsV/PNOUke7PID5HPlnxTLyQCcfOQ/JfLCRkSLOY=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y=
@@ -457,12 +459,12 @@ k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3d
 k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I=
 k8s.io/streaming v0.37.0 h1:iPBUZLZiKt5bV+lxJurASMOV07VuBhNpiwJt2//AWrM=
 k8s.io/streaming v0.37.0/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU=
-k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs=
-k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
+k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE=
+k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c=
-sigs.k8s.io/controller-runtime v0.25.0 h1:44KgRUPew331KSJpNu8zJow3iTR5W0p/SfrHdw3lV40=
-sigs.k8s.io/controller-runtime v0.25.0/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc=
+sigs.k8s.io/controller-runtime v0.25.1 h1:BKgU9OeE8xv8EbbM8cY0NVzTQs35rokkdq1jh12fMb4=
+sigs.k8s.io/controller-runtime v0.25.1/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc=
 sigs.k8s.io/e2e-framework v0.7.0 h1:AHkySTC6MvnnMbVSxaO4z1m2MhQKNFP+2Ihs5pRNLlM=
 sigs.k8s.io/e2e-framework v0.7.0/go.mod h1:1ZgXkUSjmnf18/JgHZNEATWjv48O5lJm9aI1QIsRdbw=
 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=

pkg/data-handler/posture/posture.go

@@ -47,6 +47,15 @@ type Readiness struct {
 	Message string
 }
 
+// reasonAwaitingRegistration is the readiness reason carried by a managed pod
+// that has no corresponding pooler in the shard topology.
+//
+// Every managed pod is seeded with it and only overwritten once a topology
+// entry matches. Note this is NOT what Result.Incomplete reports: that covers
+// an unreachable cell, a topology entry with no matching pod, or an UNKNOWN
+// posture, all of which are the opposite direction.
+const reasonAwaitingRegistration = "AwaitingRegistration"
+
 // Evaluate compares each managed pooler's observed postgres state with its
 // topology role. It returns nil when topology contains no active poolers, as
 // during bootstrap.
@@ -61,7 +70,7 @@ func Evaluate(
 	readiness := make(map[string]Readiness, len(managedPodNames))
 	for _, podName := range managedPodNames {
 		readiness[podName] = Readiness{
-			Reason:  "AwaitingRegistration",
+			Reason:  reasonAwaitingRegistration,
 			Message: "pooler has not registered in the shard topology",
 		}
 	}