diff --git a/cmd/agentmesh-bench/main.go b/cmd/agentmesh-bench/main.go index 3968e77b..01b69d1a 100644 --- a/cmd/agentmesh-bench/main.go +++ b/cmd/agentmesh-bench/main.go @@ -282,7 +282,6 @@ func newJoinCmd() *cobra.Command { flags.DurationVar(&opts.CallTimeout, "call-timeout", 120*time.Second, "Bound on one member's discovery and first call, from readiness") flags.DurationVar(&opts.Hold, "hold", 0, "Keep the fleet resident this long after the journeys end, sampling readiness; 0 tears it down at once") flags.DurationVar(&opts.SampleInterval, "sample-interval", 10*time.Second, "How often each resident member's readiness is read during --hold") - flags.IntVar(&opts.MetricsBasePort, "metrics-base-port", 20000, "Member i serves /readyz on 127.0.0.1:(port+i)") flags.StringArrayVar(&opts.NodeArgs, "node-arg", nil, "Extra argument for every agentmesh-node, repeatable (e.g. --node-arg=--insecure-control-plane)") flags.StringVar(&marker, "resident-marker", "", "File to create once the fleet is resident and the hold has begun") flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable") diff --git a/internal/bench/join.go b/internal/bench/join.go index 91eceb64..ac7904eb 100644 --- a/internal/bench/join.go +++ b/internal/bench/join.go @@ -93,10 +93,6 @@ type JoinOptions struct { // SampleInterval is how often each resident member is read during Hold. SampleInterval time.Duration - // MetricsBasePort is the first of Count consecutive loopback ports; - // member i serves /readyz on MetricsBasePort+i. - MetricsBasePort int - // NodeArgs are appended to every member's command line as given. NodeArgs []string @@ -208,6 +204,7 @@ const ( // and one that takes a minute is not asked a thousand times. readyPollMin = 50 * time.Millisecond readyPollMax = 500 * time.Millisecond + readyTimeout = 2 * time.Second callRetryInterval = 5 * time.Second requestTimeout = 30 * time.Second teardownGrace = 10 * time.Second @@ -222,17 +219,19 @@ const ( const mcpInitialize = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"agentmesh-bench","version":"0"}}}` type member struct { - index int - dir string - socket string - metricsAddr string - readyz string - logPath string - cmd *exec.Cmd - logFile *os.File - started time.Time - exited chan struct{} - exitErr error + index int + dir string + socket string + // client speaks to the member over its Unix socket: readiness, discovery + // and the first call. A member binds no loopback port, so a fleet of any + // size cannot collide with the kernel's ephemeral range. + client *http.Client + logPath string + cmd *exec.Cmd + logFile *os.File + started time.Time + exited chan struct{} + exitErr error readyAt time.Time calledAt time.Time @@ -364,9 +363,6 @@ func (o *JoinOptions) setDefaults() { if o.SampleInterval <= 0 { o.SampleInterval = 10 * time.Second } - if o.MetricsBasePort <= 0 { - o.MetricsBasePort = 20000 - } } func (o *JoinOptions) logf(format string, args ...any) { @@ -378,14 +374,13 @@ func (o *JoinOptions) logf(format string, args ...any) { func newMember(opts JoinOptions, i int) *member { dir := filepath.Join(opts.Dir, fmt.Sprintf("member-%04d", i)) - metricsAddr := fmt.Sprintf("127.0.0.1:%d", opts.MetricsBasePort+i) + socket := filepath.Join(dir, "node.sock") return &member{ - index: i, - dir: dir, - socket: filepath.Join(dir, "node.sock"), - metricsAddr: metricsAddr, - readyz: "http://" + metricsAddr + "/readyz", - logPath: filepath.Join(dir, "node.log"), + index: i, + dir: dir, + socket: socket, + client: unixClient(socket), + logPath: filepath.Join(dir, "node.log"), } } @@ -407,7 +402,6 @@ func (m *member) start(opts JoinOptions) error { "--data-dir", m.dir, "--socket-path", m.socket, "--bind-addr=", - "--metrics-addr", m.metricsAddr, "--listen", "/ip4/0.0.0.0/tcp/0", "--listen", "/ip4/0.0.0.0/udp/0/quic-v1", } @@ -456,11 +450,10 @@ func journey(ctx context.Context, opts JoinOptions, m *member) { // waitReady polls the member's /readyz, which answers 200 only once the node // holds an authenticated connection to a router. func (m *member) waitReady(ctx context.Context, timeout time.Duration) error { - client := &http.Client{Timeout: 2 * time.Second} deadline := time.Now().Add(timeout) interval := readyPollMin for { - if isReady(ctx, client, m.readyz) { + if m.isReady(ctx) { m.readyAt = time.Now() return nil } @@ -481,12 +474,14 @@ func (m *member) waitReady(ctx context.Context, timeout time.Duration) error { } } -func isReady(ctx context.Context, client *http.Client, url string) bool { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) +func (m *member) isReady(ctx context.Context) bool { + ctx, cancel := context.WithTimeout(ctx, readyTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost/readyz", nil) if err != nil { return false } - resp, err := client.Do(req) + resp, err := m.client.Do(req) if err != nil { return false } @@ -500,7 +495,7 @@ func isReady(ctx context.Context, client *http.Client, url string) bool { // deadline: a provider record can name a peer a rollout just replaced, and a // member that joined seconds ago has a thin routing table. func (m *member) firstCall(ctx context.Context, opts JoinOptions) error { - client := unixClient(m.socket) + client := m.client defer client.CloseIdleConnections() deadline := m.readyAt.Add(opts.CallTimeout) lastErr := "none discovered" @@ -691,12 +686,11 @@ func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReport report := &HoldReport{} var outages []time.Duration - client := &http.Client{Timeout: 2 * time.Second} start := time.Now() end := start.Add(opts.Hold) for { now := time.Now() - ready := sampleReady(ctx, client, members) + ready := sampleReady(ctx, members) count := 0 for i, m := range members { if ready[i] { @@ -759,7 +753,7 @@ func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReport } // sampleReady reads every member's /readyz, a bounded number at a time. -func sampleReady(ctx context.Context, client *http.Client, members []*member) []bool { +func sampleReady(ctx context.Context, members []*member) []bool { ready := make([]bool, len(members)) sem := make(chan struct{}, sampleConcurrency) var wg sync.WaitGroup @@ -772,7 +766,7 @@ func sampleReady(ctx context.Context, client *http.Client, members []*member) [] go func() { defer wg.Done() defer func() { <-sem }() - ready[i] = isReady(ctx, client, m.readyz) + ready[i] = m.isReady(ctx) }() } wg.Wait() diff --git a/internal/bench/join_test.go b/internal/bench/join_test.go index 1ee3306a..1191efb1 100644 --- a/internal/bench/join_test.go +++ b/internal/bench/join_test.go @@ -31,8 +31,11 @@ import ( // The members RunJoin starts are whatever binary it is pointed at. Here that // is this test binary again, re-entered through TestMain as a node that -// answers /readyz and the two sidecar routes the journey uses, with its -// behaviour set through the environment the children inherit. +// answers /readyz and the two sidecar routes the journey uses on its Unix +// socket, with its behaviour set through the environment the children +// inherit. Like the real node it binds no loopback port unless told to, and +// it refuses to be told: a fleet of a thousand on one host collided with the +// kernel's ephemeral range when every member had one. func TestMain(m *testing.M) { if os.Getenv("AGENTMESH_BENCH_FAKE_NODE") == "1" { @@ -42,11 +45,14 @@ func TestMain(m *testing.M) { } func fakeNode(args []string) int { - var metricsAddr, socket, tokenPath string + var socket, tokenPath string for i := 0; i < len(args)-1; i++ { switch args[i] { - case "--metrics-addr": - metricsAddr = args[i+1] + case "--metrics-addr", "--bind-addr": + if args[i+1] != "" { + fmt.Fprintln(os.Stderr, "fake node: a member must not bind a loopback port:", args[i], args[i+1]) + return 4 + } case "--socket-path": socket = args[i+1] case "--bootstrap-token-path": @@ -78,15 +84,14 @@ func fakeNode(args []string) int { return true } - metrics := http.NewServeMux() - metrics.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + sidecar := http.NewServeMux() + sidecar.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { if !ready() { http.Error(w, "not ready", http.StatusServiceUnavailable) return } w.WriteHeader(http.StatusOK) }) - sidecar := http.NewServeMux() sidecar.HandleFunc("/mesh/service/discover", func(w http.ResponseWriter, _ *http.Request) { var out []map[string]string for _, p := range providers { @@ -104,17 +109,11 @@ func fakeNode(args []string) int { _, _ = fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-06-18","serverInfo":{"name":"fake","version":"0"}}}`) }) - tcp, err := net.Listen("tcp", metricsAddr) - if err != nil { - fmt.Fprintln(os.Stderr, "fake node:", err) - return 1 - } unix, err := net.Listen("unix", socket) if err != nil { fmt.Fprintln(os.Stderr, "fake node:", err) return 1 } - go func() { _ = http.Serve(tcp, metrics) }() //nolint:gosec // a test double go func() { _ = http.Serve(unix, sidecar) }() //nolint:gosec // a test double stop := make(chan os.Signal, 1) signal.Notify(stop, syscall.SIGTERM, os.Interrupt) @@ -152,37 +151,9 @@ func fakeFleet(t *testing.T, count int, env map[string]string) JoinOptions { ControlPlane: "http://127.0.0.1:1", BootstrapTokenPath: token, Service: "everything", - MetricsBasePort: freePortBase(t, count), } } -// freePortBase finds count consecutive free loopback ports. -func freePortBase(t *testing.T, count int) int { - t.Helper() - for attempt := 0; attempt < 20; attempt++ { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - base := l.Addr().(*net.TCPAddr).Port - _ = l.Close() - free := true - for i := 1; i < count; i++ { - probe, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i)) - if err != nil { - free = false - break - } - _ = probe.Close() - } - if free { - return base - } - } - t.Fatal("no run of free ports") - return 0 -} - func TestRunJoinTimesEachStageAndRetriesStaleProviders(t *testing.T) { // A provider record that names a peer which does not answer is the // normal state of a mesh just after a rollout. The journey has to get diff --git a/internal/controlplane/key_rotation_test.go b/internal/controlplane/key_rotation_test.go new file mode 100644 index 00000000..6a7bc723 --- /dev/null +++ b/internal/controlplane/key_rotation_test.go @@ -0,0 +1,101 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "testing" + "time" +) + +func TestKeyRotationCheckInterval(t *testing.T) { + for _, tc := range []struct { + interval, want time.Duration + }{ + {24 * time.Hour, time.Minute}, + {time.Hour, time.Minute}, + {5 * time.Minute, 30 * time.Second}, + {2 * time.Second, 200 * time.Millisecond}, + } { + if got := keyRotationCheckInterval(tc.interval); got != tc.want { + t.Errorf("keyRotationCheckInterval(%s) = %s, want %s", tc.interval, got, tc.want) + } + } +} + +// The deadline is the store's, not the process's: a rotation that was due +// while no replica was up happens on the first check after a start, and a +// replica that restarts before the deadline does not push it back. +func TestRotateKeysIfDueFollowsTheStoredDeadline(t *testing.T) { + store := newMetricsTestStore(t) + ctx := context.Background() + interval := 24 * time.Hour + issuer, _ := startCustomMockOIDC(t) + srv, err := NewServer(Options{ + DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: issuer, + KeyRotationInterval: interval, + }, store) + if err != nil { + t.Fatal(err) + } + defer func() { _ = srv.Close() }() + if err := srv.Init(); err != nil { + t.Fatal(err) + } + keys := func() int { + t.Helper() + ks, err := store.GetAllValidKeys(ctx) + if err != nil { + t.Fatal(err) + } + return len(ks) + } + if got := keys(); got != 1 { + t.Fatalf("after Init the keyring holds %d keys, want 1", got) + } + + // Init scheduled the first rotation an interval from now; the loop it + // started checks every minute, these checks are direct. + next, err := store.NextKeyRotation(ctx) + if err != nil { + t.Fatal(err) + } + if next.IsZero() || time.Until(next) < interval-time.Minute { + t.Fatalf("first rotation scheduled at %s, want about %s from now", next, interval) + } + + srv.rotateKeysIfDue(time.Now()) + if got := keys(); got != 1 { + t.Fatalf("a check before the deadline rotated: %d keys", got) + } + + // Five days later, as after a week of restarts that never reached a tick. + late := next.Add(5 * 24 * time.Hour) + srv.rotateKeysIfDue(late) + if got := keys(); got != 2 { + t.Fatalf("a check past the deadline did not rotate: %d keys, want 2", got) + } + next, err = store.NextKeyRotation(ctx) + if err != nil { + t.Fatal(err) + } + if !next.Equal(late.Add(interval).Truncate(time.Millisecond)) { + t.Fatalf("deadline after the late rotation = %s, want %s", next, late.Add(interval)) + } + srv.rotateKeysIfDue(late.Add(time.Minute)) + if got := keys(); got != 2 { + t.Fatalf("a check right after a rotation rotated again: %d keys", got) + } +} diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.go index 29adf675..47d382dd 100644 --- a/internal/controlplane/metrics.go +++ b/internal/controlplane/metrics.go @@ -113,6 +113,9 @@ type meshSnapshot struct { // the retiring one stops verifying; zero when there is none. signingKeys int retiringKeyExpiry time.Time + // nextKeyRotation is when the store says the next rotation is due; a + // value in the past is a rotation no replica has performed. + nextKeyRotation time.Time // storageRows is the row count of every table the control plane keeps // bounded; a table that grows past its retention is a reaper that // stopped working. @@ -145,6 +148,7 @@ type meshStateCollector struct { meshPeersDesc *prometheus.Desc signingKeysDesc *prometheus.Desc retiringKeyDesc *prometheus.Desc + nextRotationDesc *prometheus.Desc storageRowsDesc *prometheus.Desc scrapeOKDesc *prometheus.Desc scrapeSampleDesc *prometheus.Desc @@ -200,6 +204,10 @@ func newMeshStateCollector(store storage.Store) *meshStateCollector { "agentmesh_control_plane_retiring_key_expiry_timestamp_seconds", "Unix time the retiring signing key stops verifying; absent when no rotation is in its grace period", nil, nil), + nextRotationDesc: prometheus.NewDesc( + "agentmesh_control_plane_next_key_rotation_timestamp_seconds", + "Unix time the next signing key rotation is due; in the past when it is overdue", + nil, nil), storageRowsDesc: prometheus.NewDesc( "agentmesh_control_plane_storage_rows", "Rows in a control plane store table", @@ -219,7 +227,7 @@ func (c *meshStateCollector) Describe(ch chan<- *prometheus.Desc) { for _, d := range []*prometheus.Desc{ c.nodesDesc, c.usersDesc, c.requestsDesc, c.tokensDesc, c.routersDesc, c.routerPeersDesc, c.routerDHTDesc, c.routerLeaseDesc, c.meshPeersDesc, - c.signingKeysDesc, c.retiringKeyDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc, + c.signingKeysDesc, c.retiringKeyDesc, c.nextRotationDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc, } { ch <- d } @@ -266,6 +274,9 @@ func (c *meshStateCollector) Collect(ch chan<- prometheus.Metric) { if !s.retiringKeyExpiry.IsZero() { ch <- prometheus.MustNewConstMetric(c.retiringKeyDesc, prometheus.GaugeValue, float64(s.retiringKeyExpiry.Unix())) } + if !s.nextKeyRotation.IsZero() { + ch <- prometheus.MustNewConstMetric(c.nextRotationDesc, prometheus.GaugeValue, float64(s.nextKeyRotation.Unix())) + } for table, n := range s.storageRows { ch <- prometheus.MustNewConstMetric(c.storageRowsDesc, prometheus.GaugeValue, float64(n), table) } @@ -312,12 +323,14 @@ func readMeshSnapshot(ctx context.Context, store storage.Store, now time.Time) ( } snap.users = len(users) - reqs, err := store.ListEnrollmentRequests(ctx) + // A count, not a list: the table holds one row per enrollment within the + // retention, tens of thousands on a busy mesh, and this runs every 30s. + requests, err := store.CountEnrollmentRequests(ctx) if err != nil { return snap, err } - for _, r := range reqs { - snap.enrollmentRequests[enrollmentStatusLabel(r.Status)]++ + for status, n := range requests { + snap.enrollmentRequests[enrollmentStatusLabel(status)] += int(n) } tokens, err := store.ListBootstrapTokens(ctx) @@ -351,6 +364,10 @@ func readMeshSnapshot(ctx context.Context, store storage.Store, now time.Time) ( if err != nil { return snap, err } + snap.nextKeyRotation, err = store.NextKeyRotation(ctx) + if err != nil { + return snap, err + } return snap, nil } diff --git a/internal/controlplane/metrics_test.go b/internal/controlplane/metrics_test.go index 099d97d7..d066d760 100644 --- a/internal/controlplane/metrics_test.go +++ b/internal/controlplane/metrics_test.go @@ -344,6 +344,7 @@ func TestMetricsEndpointServesMeshStateAndRuntime(t *testing.T) { "agentmesh_control_plane_routers_active 0", "agentmesh_control_plane_mesh_connected_peers 0", `agentmesh_control_plane_storage_rows{table="nodes"} 0`, + "agentmesh_control_plane_next_key_rotation_timestamp_seconds ", `agentmesh_control_plane_storage_rows{table="enrollment_requests"} 0`, "agentmesh_control_plane_catalog_entries ", `agentmesh_control_plane_storage_rows_deleted_total{table="routers"} `, diff --git a/internal/controlplane/node_gc.go b/internal/controlplane/node_gc.go index 22ffd902..3723e34f 100644 --- a/internal/controlplane/node_gc.go +++ b/internal/controlplane/node_gc.go @@ -113,7 +113,13 @@ func (s *Server) runNodeGCLoop() { // grow. func (s *Server) gcStorage(now time.Time) { s.gcExpiredNodes(now) + if s.ctx.Err() != nil { + return + } s.gcEnrollmentRecords(now) + if s.ctx.Err() != nil { + return + } s.pruneCatalog(now) } diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go index a535d4ea..9b4537a4 100644 --- a/internal/controlplane/server.go +++ b/internal/controlplane/server.go @@ -300,6 +300,19 @@ func (s *Server) Init() error { } else if err != nil { return fmt.Errorf("failed to query initial keyring status: %w", err) } + // The rotation deadline lives in the store, shared by the replicas and + // kept across restarts; a store that never had one gets its first an + // interval from now. + if interval := s.config.KeyRotationInterval; interval > 0 { + first := time.Now().Add(interval) + scheduled, err := s.store.ScheduleKeyRotationIfUnset(ctx, first) + if err != nil { + return fmt.Errorf("failed to schedule the first key rotation: %w", err) + } + if scheduled { + logger.Infof("First key rotation scheduled at %s", first.UTC().Format(time.RFC3339)) + } + } // Bootstrap Policy is now disabled; starting default closed. @@ -514,64 +527,80 @@ func (s *Server) getProviders() map[string]*oidc.Provider { return pCopy } +// keyRotationCheckInterval is how often a replica asks the store whether a +// rotation is due: a tenth of the interval, at most once a minute. The +// deadline itself lives in the store, so a replica that restarts more often +// than the interval still rotates on time. +func keyRotationCheckInterval(interval time.Duration) time.Duration { + return min(time.Minute, interval/10) +} + +// runKeyRotationLoop rotates the signing keys when the deadline the store +// holds (set by Init, advanced by each rotation) has passed. The check runs +// well within the interval, so a control plane that restarts more often +// than it rotates still rotates on time instead of never. func (s *Server) runKeyRotationLoop() { defer s.wg.Done() - if s.config.KeyRotationInterval <= 0 { + interval := s.config.KeyRotationInterval + if interval <= 0 { return } - ticker := time.NewTicker(s.config.KeyRotationInterval) + ticker := time.NewTicker(keyRotationCheckInterval(interval)) defer ticker.Stop() for { select { case <-ticker.C: - // Replicas share one DB but tick independently; only the replica - // that wins this claim may rotate for the current window. - now := time.Now() - claimed, err := s.store.ClaimKeyRotation(s.ctx, now, s.config.KeyRotationInterval) - if err != nil { - logger.Errorf("Failed to claim key rotation window: %v", err) - continue - } - if !claimed { - logger.Debug("Skipping key rotation: another replica already claimed this window") - continue - } - - logger.Info("Rotating Biscuit signing keys...") - newPub, newPriv, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - logger.Errorf("Failed to generate key pair for rotation: %v", err) - // Give up the window so a retry isn't stuck waiting a full interval. - if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, s.config.KeyRotationInterval); relErr != nil { - logger.Errorf("Failed to release key rotation claim: %v", relErr) - } - continue - } - err = s.store.RotateKeys(s.ctx, newPriv, newPub, s.config.KeyGracePeriod) - if err != nil { - logger.Errorf("Failed to rotate keyring: %v", err) - // Same as above: don't let a failed rotation strand the mesh - // on unrotated keys until the next full interval. - if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, s.config.KeyRotationInterval); relErr != nil { - logger.Errorf("Failed to release key rotation claim: %v", relErr) - } - } else { - logger.Infof("Key rotation committed. New current public key: %s", hex.EncodeToString(newPub)) - if _, oidcErr := s.RotateOIDCKey(s.config.KeyGracePeriod); oidcErr != nil { - logger.Warnf("Failed to rotate OIDC signing key: %v", oidcErr) - } - if err := s.getMeshAdapter().PublishEvent(s.ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil { - logger.Warnf("Failed to publish KEY_ROTATION event to mesh: %v", err) - } - } + s.rotateKeysIfDue(time.Now()) case <-s.ctx.Done(): return } } } +// rotateKeysIfDue claims the rotation window and rotates when it is due. +// Replicas share one store and check independently; only the one that wins +// the claim rotates for this window. +func (s *Server) rotateKeysIfDue(now time.Time) { + interval := s.config.KeyRotationInterval + claimed, err := s.store.ClaimKeyRotation(s.ctx, now, interval) + if err != nil { + logger.Errorf("Failed to claim key rotation window: %v", err) + return + } + if !claimed { + return + } + + logger.Info("Rotating Biscuit signing keys...") + newPub, newPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + logger.Errorf("Failed to generate key pair for rotation: %v", err) + // Give up the window so a retry isn't stuck waiting a full interval. + if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, interval); relErr != nil { + logger.Errorf("Failed to release key rotation claim: %v", relErr) + } + return + } + if err := s.store.RotateKeys(s.ctx, newPriv, newPub, s.config.KeyGracePeriod); err != nil { + logger.Errorf("Failed to rotate keyring: %v", err) + // Same as above: don't let a failed rotation strand the mesh + // on unrotated keys until the next full interval. + if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, interval); relErr != nil { + logger.Errorf("Failed to release key rotation claim: %v", relErr) + } + return + } + logger.Infof("Key rotation committed. New current public key: %s; next rotation at %s", hex.EncodeToString(newPub), now.Add(interval).UTC().Format(time.RFC3339)) + if _, oidcErr := s.RotateOIDCKey(s.config.KeyGracePeriod); oidcErr != nil { + logger.Warnf("Failed to rotate OIDC signing key: %v", oidcErr) + } + if err := s.getMeshAdapter().PublishEvent(s.ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil { + logger.Warnf("Failed to publish KEY_ROTATION event to mesh: %v", err) + } +} + // HandleHealthz HTTP GET `/healthz` func (s *Server) HandleHealthz(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { diff --git a/internal/node/node.go b/internal/node/node.go index 34128de9..8be81b97 100644 --- a/internal/node/node.go +++ b/internal/node/node.go @@ -1302,13 +1302,19 @@ func (n *AgentMeshNode) authenticateRouterSession(ctx context.Context, router pe return nil } +// readmitRetryDelays spaces the retries of a re-admission a router refused. +// A credential refreshed right after a key rotation is signed by a key the +// router may not have synced yet; it has within a few seconds, so the node +// asks again instead of waiting a monitor interval with no router. +var readmitRetryDelays = []time.Duration{500 * time.Millisecond, time.Second, 2 * time.Second} + // readmitRouters presents the current credential to every router this node // holds an authenticated session with. A router admits a peer until the // biscuit it was shown expires, whatever the connection does, so a refreshed // credential must be shown on the live session or relay circuits to this -// node are refused once the old one lapses. A router that refuses is -// dropped from the authenticated set; the connection monitor then dials and -// handshakes afresh. +// node are refused once the old one lapses. A router that still refuses +// after the retries is dropped from the authenticated set; the connection +// monitor then dials and handshakes afresh. func (n *AgentMeshNode) readmitRouters(ctx context.Context) { if n.Host == nil { return @@ -1318,16 +1324,36 @@ func (n *AgentMeshNode) readmitRouters(ctx context.Context) { return } for _, router := range n.authenticatedRouterIDs() { + n.readmitRouter(ctx, router, biscuitBytes) + } +} + +func (n *AgentMeshNode) readmitRouter(ctx context.Context, router peer.ID, biscuitBytes []byte) { + var err error + for attempt := 0; ; attempt++ { routerCtx, cancel := context.WithTimeout(ctx, n.config.RouterConnectTimeout) - err := n.authenticateRouterSession(routerCtx, router, biscuitBytes) + err = n.authenticateRouterSession(routerCtx, router, biscuitBytes) cancel() - if err != nil { - logger.Warnf("[AuthN] Router %s did not accept the refreshed credential; the connection monitor will re-handshake: %v", router, err) - n.mu.Lock() - delete(n.authenticatedRouters, router) - n.mu.Unlock() + if err == nil { + if attempt > 0 { + logger.Infow("[AuthN] Router accepted the refreshed credential on retry", "router", router, "attempt", attempt+1) + } + return + } + if attempt >= len(readmitRetryDelays) { + break + } + logger.Warnw("[AuthN] Router did not accept the refreshed credential; retrying", "router", router, "attempt", attempt+1, "retry_in", readmitRetryDelays[attempt], "error", err) + select { + case <-ctx.Done(): + return + case <-time.After(readmitRetryDelays[attempt]): } } + logger.Warnw("[AuthN] Router did not accept the refreshed credential; the connection monitor will re-handshake", "router", router, "attempts", len(readmitRetryDelays)+1, "error", err) + n.mu.Lock() + delete(n.authenticatedRouters, router) + n.mu.Unlock() } func (n *AgentMeshNode) performRouterAuthHandshake(s network.Stream, biscuitBytes []byte, expectedRouter peer.ID) (bool, error) { diff --git a/internal/node/readmit_test.go b/internal/node/readmit_test.go new file mode 100644 index 00000000..8f55fa59 --- /dev/null +++ b/internal/node/readmit_test.go @@ -0,0 +1,145 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node + +import ( + "context" + "crypto/ed25519" + "sync/atomic" + "testing" + "time" + + "github.com/eclipse-biscuit/biscuit-go/v2" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-msgio" + "google.golang.org/protobuf/proto" + + "github.com/google/agentmesh/api" +) + +// A router that refuses the first handshakes, as one does when a node shows a +// credential under a key the router has not synced yet, then accepts. +func startRefusingRouter(t *testing.T, cpPriv ed25519.PrivateKey, refusals int32) (host.Host, *atomic.Int32) { + t.Helper() + h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = h.Close() }) + builder := biscuit.NewBuilder(cpPriv) + for _, f := range []biscuit.Fact{ + {Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(h.ID().String())}}}, + {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleRouter)}}}, + {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(24 * time.Hour))}}}, + api.MarkerFact(api.FactTargetUnrestricted), + } { + if err := builder.AddAuthorityFact(f); err != nil { + t.Fatal(err) + } + } + tok, err := builder.Build() + if err != nil { + t.Fatal(err) + } + routerBiscuit, err := tok.Serialize() + if err != nil { + t.Fatal(err) + } + var handshakes atomic.Int32 + h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) { + defer func() { _ = s.Close() }() + reader := msgio.NewVarintReaderSize(s, 1024*64) + msg, err := reader.ReadMsg() + if err != nil { + return + } + reader.ReleaseMsg(msg) + resp := &api.AuthResponse{Success: true, Biscuit: routerBiscuit} + if handshakes.Add(1) <= refusals { + resp = &api.AuthResponse{Success: false, Error: "no valid key found for verification"} + } + data, err := proto.Marshal(resp) + if err != nil { + return + } + _ = msgio.NewVarintWriter(s).WriteMsg(data) + }) + return h, &handshakes +} + +// newReadmitTestNode is a node holding an authenticated session with router, +// connected to it, with a refreshed credential to show. +func newReadmitTestNode(t *testing.T, cpPub ed25519.PublicKey, router host.Host) *AgentMeshNode { + t.Helper() + h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = h.Close() }) + if err := h.Connect(t.Context(), peer.AddrInfo{ID: router.ID(), Addrs: router.Addrs()}); err != nil { + t.Fatal(err) + } + n := &AgentMeshNode{ + Host: h, + BiscuitTimeout: 2 * time.Second, + authenticatedRouters: map[peer.ID]bool{router.ID(): true}, + trustedKeys: []TrustedKey{{Key: cpPub, ReceivedAt: time.Now()}}, + } + n.config.RouterConnectTimeout = 2 * time.Second + n.cachedIdentity.Store([]byte("refreshed-credential")) + return n +} + +// TestReadmitRetriesARouterThatHasNotSyncedTheNewKey pins the window after a +// key rotation: the node refreshes under the new key and shows it to its +// routers at once; a router that has not synced the key yet refuses. The +// node asks again within seconds and keeps the session, instead of dropping +// the router and serving 503 until the connection monitor's next minute. +func TestReadmitRetriesARouterThatHasNotSyncedTheNewKey(t *testing.T) { + cpPub, cpPriv, _ := ed25519.GenerateKey(nil) + router, handshakes := startRefusingRouter(t, cpPriv, 2) + n := newReadmitTestNode(t, cpPub, router) + + start := time.Now() + n.readmitRouters(t.Context()) + if got := handshakes.Load(); got != 3 { + t.Fatalf("router saw %d handshakes, want 3: two refusals and the one that succeeded", got) + } + if !n.IsConnected() { + t.Fatal("the router accepted the third handshake, but the node dropped it from the authenticated set") + } + if elapsed := time.Since(start); elapsed > 4*time.Second { + t.Errorf("re-admission took %s; the retries should fit in a few seconds", elapsed) + } +} + +// A router that keeps refusing is dropped after the retries, so the monitor +// dials and handshakes afresh, as before. +func TestReadmitDropsARouterThatKeepsRefusing(t *testing.T) { + cpPub, cpPriv, _ := ed25519.GenerateKey(nil) + router, handshakes := startRefusingRouter(t, cpPriv, 100) + n := newReadmitTestNode(t, cpPub, router) + + n.readmitRouters(context.Background()) + if got, want := handshakes.Load(), int32(len(readmitRetryDelays)+1); got != want { + t.Fatalf("router saw %d handshakes, want %d", got, want) + } + if n.IsConnected() { + t.Fatal("a router that refused every handshake is still in the authenticated set") + } +} diff --git a/internal/node/sidecar.go b/internal/node/sidecar.go index 79f3fd7b..00ba4b40 100644 --- a/internal/node/sidecar.go +++ b/internal/node/sidecar.go @@ -47,7 +47,7 @@ func StartSidecarServer(node *AgentMeshNode, addr, socketPath, token, certFile, // Public endpoints mux.HandleFunc("/healthz", handleHealthz) - mux.HandleFunc("/readyz", handleReadyz) + mux.HandleFunc("/readyz", handleReadyzFor(node)) mux.HandleFunc("/.well-known/oauth-protected-resource", func(w http.ResponseWriter, r *http.Request) { handleOAuthProtectedResource(node, w, r) }) @@ -315,7 +315,7 @@ func StartUnauthSidecarServer(controlPlaneURL, addr, socketPath, certFile, keyFi // Public endpoints mux.HandleFunc("/healthz", handleHealthz) - mux.HandleFunc("/readyz", handleReadyz) + mux.HandleFunc("/readyz", handleReadyzFor(nil)) // Mount Unauthenticated MCP handler mcpHandler := NewUnauthenticatedMCPHandler(controlPlaneURL) @@ -383,10 +383,20 @@ func handleHealthz(w http.ResponseWriter, r *http.Request) { } } -func handleReadyz(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - if _, err := w.Write([]byte("OK")); err != nil { - logger.Errorf("Failed to write response: %v", err) +// handleReadyzFor answers 200 only once the node holds an authenticated +// connection to a router, the same answer the metrics listener gives, so a +// probe or a harness reading either listener learns the same thing. A node +// without an identity yet (nil) is not ready. +func handleReadyzFor(node *AgentMeshNode) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + if node == nil || node.debugReady() != nil || !node.IsConnected() { + http.Error(w, "not connected to the mesh", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + if _, err := w.Write([]byte("OK")); err != nil { + logger.Errorf("Failed to write response: %v", err) + } } } diff --git a/internal/node/sidecar_auth_test.go b/internal/node/sidecar_auth_test.go index b3346524..34405b53 100644 --- a/internal/node/sidecar_auth_test.go +++ b/internal/node/sidecar_auth_test.go @@ -19,6 +19,9 @@ import ( "path/filepath" "testing" "time" + + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" ) // TestConstantTimeEqual is a correctness check, not a timing one: a timing @@ -57,6 +60,80 @@ func TestConstantTimeEqual(t *testing.T) { } } +// TestSidecarReadyzFollowsTheMeshConnection pins that the sidecar's /readyz +// says what the metrics listener's does: 503 until the node holds an +// authenticated, connected router, then 200, on the socket and on TCP. A +// harness that reads readiness over the socket, as agentmesh-bench does so +// that a member binds no loopback port, learns the same thing a kubelet +// probing the metrics port would. +func TestSidecarReadyzFollowsTheMeshConnection(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = store.Close() }() + host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = host.Close() }() + node := &AgentMeshNode{ + Host: host, + Store: store, + BiscuitTimeout: 500 * time.Millisecond, + services: NewServiceRegistry(&fakeDHT{}, 0), + } + node.Directory = &routerDirectory{host: host, routers: node.attachedRouters, addrs: host.Addrs} + socketPath := filepath.Join(t.TempDir(), "agentmesh.sock") + srv, err := StartSidecarServer(node, "127.0.0.1:0", socketPath, "test-token", "", "", "") + if err != nil { + t.Fatalf("failed to start sidecar server: %v", err) + } + defer func() { _ = srv.Close() }() + socket := waitForSocket(t, socketPath) + tcp := &http.Client{Timeout: 2 * time.Second} + + readyz := func(t *testing.T, client *http.Client, url string) int { + t.Helper() + resp, err := client.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode + } + for name, url := range map[string]string{"socket": "http://localhost/readyz", "tcp": "http://" + node.BoundHTTPAddr + "/readyz"} { + client := socket + if name == "tcp" { + client = tcp + } + if code := readyz(t, client, url); code != http.StatusServiceUnavailable { + t.Errorf("%s /readyz = %d with no authenticated router, want 503", name, code) + } + } + + routerHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = routerHost.Close() }() + if err := host.Connect(t.Context(), peer.AddrInfo{ID: routerHost.ID(), Addrs: routerHost.Addrs()}); err != nil { + t.Fatal(err) + } + node.mu.Lock() + node.authenticatedRouters = map[peer.ID]bool{routerHost.ID(): true} + node.mu.Unlock() + for name, url := range map[string]string{"socket": "http://localhost/readyz", "tcp": "http://" + node.BoundHTTPAddr + "/readyz"} { + client := socket + if name == "tcp" { + client = tcp + } + if code := readyz(t, client, url); code != http.StatusOK { + t.Errorf("%s /readyz = %d once a router is authenticated and connected, want 200", name, code) + } + } +} + // TestMetricsGatedOnTCPButNotOnTheSocket pins where the metrics endpoint sits in // the sidecar's trust model. Its labels carry peer IDs and per-peer request // counts, and the TCP listener is reachable by any local process, so it is diff --git a/internal/node/sidecar_test.go b/internal/node/sidecar_test.go index 17b55e8a..0f25ea78 100644 --- a/internal/node/sidecar_test.go +++ b/internal/node/sidecar_test.go @@ -399,7 +399,8 @@ func TestSidecarServerAuthEnforcement(t *testing.T) { needsToken bool }{ {"healthz is public", "GET", "/healthz", http.StatusOK, false}, - {"readyz is public", "GET", "/readyz", http.StatusOK, false}, + // Public, and honest: this node holds no router, so it is not ready. + {"readyz is public", "GET", "/readyz", http.StatusServiceUnavailable, false}, // Liveness says nothing; metrics name peers and count their traffic. {"metrics is protected", "GET", "/metrics", http.StatusUnauthorized, false}, // Services are declared in configuration only: the former runtime diff --git a/internal/storage/round_trip_test.go b/internal/storage/round_trip_test.go index 721fd46f..25b2d812 100644 --- a/internal/storage/round_trip_test.go +++ b/internal/storage/round_trip_test.go @@ -17,6 +17,7 @@ package storage import ( "context" "errors" + "fmt" "reflect" "slices" "strconv" @@ -388,6 +389,80 @@ func TestDeleteStaleEnrollmentRecords(t *testing.T) { } } +func TestCountEnrollmentRequestsByStatus(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + now := time.Now() + if err := store.SaveBootstrapToken(ctx, &BootstrapToken{ID: "tok", TokenHash: "hash", Role: api.RoleNode, MaxUsages: 10, CreatedAt: now, ExpiresAt: now.Add(time.Hour)}); err != nil { + t.Fatal(err) + } + for i, status := range []api.EnrollmentStatus{ + api.EnrollmentStatus_ENROLLMENT_STATUS_PENDING, + api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED, + api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED, + api.EnrollmentStatus_ENROLLMENT_STATUS_REJECTED, + } { + if err := store.CreateEnrollmentRequest(ctx, &EnrollmentRequest{ + ID: fmt.Sprintf("req-%d", i), PeerID: fmt.Sprintf("peer-%d", i), PublicKey: []byte("pub"), TokenID: "tok", Status: status, CreatedAt: now, + }); err != nil { + t.Fatal(err) + } + } + counts, err := store.CountEnrollmentRequests(ctx) + if err != nil { + t.Fatalf("CountEnrollmentRequests: %v", err) + } + want := map[api.EnrollmentStatus]int64{ + api.EnrollmentStatus_ENROLLMENT_STATUS_PENDING: 1, + api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED: 2, + api.EnrollmentStatus_ENROLLMENT_STATUS_REJECTED: 1, + } + if len(counts) != len(want) { + t.Fatalf("counts = %v, want %v", counts, want) + } + for status, n := range want { + if counts[status] != n { + t.Errorf("%s = %d, want %d", status, counts[status], n) + } + } + // The list and the count agree, so the admin view and the gauge do. + list, err := store.ListEnrollmentRequests(ctx) + if err != nil { + t.Fatal(err) + } + if len(list) != 4 { + t.Errorf("ListEnrollmentRequests returned %d, want 4", len(list)) + } +} + +// A node written without a last-seen instant is seen now, not at the zero +// time: the silent-member sweep would otherwise collect it on the control +// plane's first sweep, which raced with every test that enrolled a node +// this way (TestUserStatusAndTenancy failed so in CI). +func TestEnrollNodeWithoutEnrolledAtIsSeenNow(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + before := time.Now().Add(-time.Second) + node := &EnrolledNode{PeerID: "fresh", PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode} + if err := store.EnrollNode(ctx, node); err != nil { + t.Fatal(err) + } + got, err := store.GetNode(ctx, "fresh") + if err != nil { + t.Fatal(err) + } + if got.EnrolledAt.Before(before) { + t.Fatalf("EnrolledAt = %s, want about now", got.EnrolledAt) + } + regular, ephemeral, err := store.DeleteSilentBootstrapNodes(ctx, time.Now().Add(-30*24*time.Hour), time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if regular != 0 || ephemeral != 0 { + t.Fatalf("the sweep deleted regular=%d ephemeral=%d rows; a node enrolled now is not silent", regular, ephemeral) + } +} + func TestDeleteExpiredRouterLeases(t *testing.T) { store := newTestStore(t) ctx := context.Background() diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go index 6ca97f52..266e7dd1 100644 --- a/internal/storage/sql_store.go +++ b/internal/storage/sql_store.go @@ -857,6 +857,31 @@ func (s *SQLStore) ReleaseKeyRotationClaim(ctx context.Context, now time.Time, i return err } +// ScheduleKeyRotationIfUnset implements Store. +func (s *SQLStore) ScheduleKeyRotationIfUnset(ctx context.Context, at time.Time) (bool, error) { + res, err := s.db.ExecContext(ctx, s.rebind(`UPDATE rotation_lock SET next_rotation_at = ? WHERE id = 1 AND next_rotation_at = 0`), at.UnixMilli()) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + if err != nil { + return false, err + } + return n == 1, nil +} + +// NextKeyRotation implements Store. +func (s *SQLStore) NextKeyRotation(ctx context.Context) (time.Time, error) { + var ms int64 + if err := s.db.QueryRowContext(ctx, `SELECT next_rotation_at FROM rotation_lock WHERE id = 1`).Scan(&ms); err != nil { + return time.Time{}, err + } + if ms == 0 { + return time.Time{}, nil + } + return time.UnixMilli(ms), nil +} + // RotateKeys implements Store. func (s *SQLStore) RotateKeys(ctx context.Context, newPriv ed25519.PrivateKey, newPub ed25519.PublicKey, gracePeriod time.Duration) error { tx, err := s.db.BeginTx(ctx, nil) @@ -902,6 +927,9 @@ func (s *SQLStore) EnrollNode(ctx context.Context, node *EnrolledNode) error { if err != nil { return fmt.Errorf("failed to marshal labels: %w", err) } + if node.EnrolledAt.IsZero() { + node.EnrolledAt = time.Now() + } var query string if s.isPostgres() { @@ -1940,6 +1968,26 @@ func (s *SQLStore) CountNodes(ctx context.Context, now time.Time) ([]NodeCount, return counts, rows.Err() } +// CountEnrollmentRequests implements Store. +func (s *SQLStore) CountEnrollmentRequests(ctx context.Context) (map[api.EnrollmentStatus]int64, error) { + rows, err := s.db.QueryContext(ctx, `SELECT status, COUNT(*) FROM enrollment_requests GROUP BY status`) + if err != nil { + return nil, fmt.Errorf("failed to count enrollment requests: %w", err) + } + defer func() { _ = rows.Close() }() + + counts := make(map[api.EnrollmentStatus]int64) + for rows.Next() { + var status int32 + var n int64 + if err := rows.Scan(&status, &n); err != nil { + return nil, fmt.Errorf("failed to scan enrollment request count: %w", err) + } + counts[api.EnrollmentStatus(status)] = n + } + return counts, rows.Err() +} + // ListBootstrapTokens retrieves all bootstrap tokens. func (s *SQLStore) ListBootstrapTokens(ctx context.Context) ([]BootstrapToken, error) { query := s.rebind(`SELECT id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at, revoked_at, autonomous_recovery, ephemeral FROM bootstrap_tokens ORDER BY created_at DESC`) diff --git a/internal/storage/sql_store_test.go b/internal/storage/sql_store_test.go index 1158a029..31040ddf 100644 --- a/internal/storage/sql_store_test.go +++ b/internal/storage/sql_store_test.go @@ -225,6 +225,69 @@ func TestClaimKeyRotation(t *testing.T) { } } +func TestScheduleKeyRotationIfUnset(t *testing.T) { + store := newTestStore(t) + defer func() { _ = store.Close() }() + ctx := context.Background() + t0 := time.Now().Truncate(time.Millisecond) + interval := 24 * time.Hour + + next, err := store.NextKeyRotation(ctx) + if err != nil { + t.Fatal(err) + } + if !next.IsZero() { + t.Fatalf("fresh store has a rotation deadline %s, want none", next) + } + + // The first replica to start schedules; a replica starting later, or + // the same one restarting, leaves the deadline alone. + scheduled, err := store.ScheduleKeyRotationIfUnset(ctx, t0.Add(interval)) + if err != nil { + t.Fatal(err) + } + if !scheduled { + t.Fatal("first schedule on a fresh store was not applied") + } + scheduled, err = store.ScheduleKeyRotationIfUnset(ctx, t0.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if scheduled { + t.Fatal("a second schedule replaced the deadline; want it kept") + } + next, err = store.NextKeyRotation(ctx) + if err != nil { + t.Fatal(err) + } + if !next.Equal(t0.Add(interval)) { + t.Fatalf("deadline = %s, want %s", next, t0.Add(interval)) + } + + // Not due before the deadline, due at it; a claim moves it one interval on. + claimed, err := store.ClaimKeyRotation(ctx, t0.Add(interval-time.Second), interval) + if err != nil { + t.Fatal(err) + } + if claimed { + t.Fatal("claim before the deadline succeeded") + } + claimed, err = store.ClaimKeyRotation(ctx, t0.Add(interval), interval) + if err != nil { + t.Fatal(err) + } + if !claimed { + t.Fatal("claim at the deadline failed") + } + next, err = store.NextKeyRotation(ctx) + if err != nil { + t.Fatal(err) + } + if !next.Equal(t0.Add(2 * interval)) { + t.Fatalf("deadline after the claim = %s, want %s", next, t0.Add(2*interval)) + } +} + func TestReleaseKeyRotationClaim(t *testing.T) { store := newTestStore(t) defer func() { _ = store.Close() }() diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 52ad4730..80ccb070 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -248,6 +248,15 @@ type Store interface { // the rotation that followed a successful claim failed. ReleaseKeyRotationClaim(ctx context.Context, now time.Time, interval time.Duration) error + // ScheduleKeyRotationIfUnset sets the first rotation deadline of a store + // that never had one, and reports whether it did. A store that already + // carries a deadline, due or not, is left as it is. + ScheduleKeyRotationIfUnset(ctx context.Context, at time.Time) (bool, error) + + // NextKeyRotation is the deadline ClaimKeyRotation will honour: the + // instant the next rotation is due, zero when none was ever scheduled. + NextKeyRotation(ctx context.Context) (time.Time, error) + // SaveInitialKey sets the initial key pair if no keys exist yet. SaveInitialKey(ctx context.Context, priv ed25519.PrivateKey, pub ed25519.PublicKey) error @@ -260,7 +269,9 @@ type Store interface { // ListUsers retrieves all registered users. ListUsers(ctx context.Context) ([]User, error) - // EnrollNode registers or updates a node enrollment. + // EnrollNode registers or updates a node enrollment. EnrolledAt is when + // the member was last seen; unset, it is recorded as now, never as the + // zero time, which the silent-member sweep would read as decades ago. EnrollNode(ctx context.Context, node *EnrolledNode) error // GetNode retrieves node enrollment details. @@ -368,6 +379,9 @@ type Store interface { // CountNodes counts every enrolled node by role and by its state at now. CountNodes(ctx context.Context, now time.Time) ([]NodeCount, error) + // CountEnrollmentRequests counts the enrollment requests by status. + CountEnrollmentRequests(ctx context.Context) (map[api.EnrollmentStatus]int64, error) + // DeleteExpiredNodes removes enrolled nodes whose session expired before // the given time and that are not banned, returning how many went. A // lapsed session already refuses every refresh, so the row only records diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md index b0d0d50f..c35086d3 100644 --- a/site/content/docs/reference/control-plane.md +++ b/site/content/docs/reference/control-plane.md @@ -149,6 +149,13 @@ with `403`). The console uses these routes. - **Bans propagate in two ways.** `/info` lists banned peer IDs for nodes and routers that start or restart. A signed mesh event reaches the ones that are already running. +- **The rotation deadline is in the database.** The first rotation is due + one `--key-rotation-interval` after the first start, each rotation sets + the next, and every replica checks the deadline at least once a minute. + A control plane that restarts more often than it rotates still rotates on + time, and one that was down when a rotation was due rotates on its first + check. `agentmesh_control_plane_next_key_rotation_timestamp_seconds` is + the deadline; a value in the past is a rotation that is not happening. - **Node records are garbage-collected.** A node whose session has expired is deleted after `--node-retention`. A node enrolled with a bootstrap token has no session; its record is deleted once it has not refreshed its diff --git a/tests/integration/member_burst_test.go b/tests/integration/member_burst_test.go index e5940169..d1a590bb 100644 --- a/tests/integration/member_burst_test.go +++ b/tests/integration/member_burst_test.go @@ -16,7 +16,6 @@ package integration_test import ( "context" - "fmt" "net" "net/http/httptest" "os" @@ -92,7 +91,6 @@ func TestMemberBurstJoinsConcurrently(t *testing.T) { Service: "everything", ReadyTimeout: 30 * time.Second, CallTimeout: 30 * time.Second, - MetricsBasePort: reservePortBlock(t, members), NodeArgs: []string{"--allow-loopback"}, Log: testLogWriter{t}, }) @@ -134,29 +132,6 @@ func TestMemberBurstJoinsConcurrently(t *testing.T) { members, report.ReadyMs.P50, report.ReadyMs.P95, report.JourneyMs.P50, report.JourneyMs.P95, report.JourneyMs.Max, report.CallAttempts) } -// reservePortBlock finds n consecutive free loopback ports in the reserved -// range and returns the first. -func reservePortBlock(t *testing.T, n int) int { - t.Helper() - for range 50 { - base := getFreePort(t) - free := true - for i := 1; i < n; i++ { - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i)) - if err != nil { - free = false - break - } - _ = l.Close() - } - if free { - return base - } - } - t.Fatalf("no run of %d free ports", n) - return 0 -} - // logTail returns the last part of a log file, for a failure message. func logTail(path string) string { data, err := os.ReadFile(path) // #nosec G304 -- a log the test wrote diff --git a/tests/integration/rotation_test.go b/tests/integration/rotation_test.go index 59c4344b..7c62fe2c 100644 --- a/tests/integration/rotation_test.go +++ b/tests/integration/rotation_test.go @@ -211,7 +211,7 @@ roles: "-args", "{}", ) if err != nil { - t.Fatalf("failed to query node: %v\nstdout: %s\nstderr: %s", err, stdout, stderr) + t.Fatalf("failed to query node: %v\nstdout: %s\nstderr: %s\n--- node.log ---\n%s", err, stdout, stderr, readLog(t, nodeLogPath)) } if !strings.Contains(stdout, "attached_routers") { t.Fatalf("unexpected node response: %s", stdout) diff --git a/tests/scale/member-journey.sh b/tests/scale/member-journey.sh index 2c67d100..ee38e309 100755 --- a/tests/scale/member-journey.sh +++ b/tests/scale/member-journey.sh @@ -259,16 +259,15 @@ summarize() { # a backgrounded function is a subshell, and a signal to it would not reach # the run inside. JOIN_CMD=() -join_cmd() { # phase count base-port extra... - local phase="$1" count="$2" base="$3" a - shift 3 +join_cmd() { # phase count extra... + local phase="$1" count="$2" a + shift 2 JOIN_CMD=("$AGENTMESH_BENCH" join --node-bin "$AGENTMESH_NODE" --control-plane "$CONTROL_PLANE" --bootstrap-token-path "$BOOTSTRAP_TOKEN_PATH" --service "$SERVICE" --count "$count" - --metrics-base-port "$base" --dir "$OUT/$phase" --label "phase=$phase" --label "count=$count" --label "burst=$COUNT" --label "host=$(hostname)" --out "$OUT/$phase.json") @@ -288,7 +287,7 @@ rm -f "$OUT/resident" snapshot before event burst-start log "burst: ${COUNT} members at ${RAMP}/s, then holding ${HOLD}" -join_cmd burst "$COUNT" 20000 --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident" +join_cmd burst "$COUNT" --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident" "${JOIN_CMD[@]}" 2> "$OUT/burst.log" & JOIN_PID=$! while [[ ! -f "$OUT/resident" ]]; do @@ -309,7 +308,7 @@ PY if (( LATE > 0 )); then event late-start log "late: ${LATE} members join the populated mesh" - join_cmd late "$LATE" 30000 + join_cmd late "$LATE" "${JOIN_CMD[@]}" 2> "$OUT/late.log" || log "late phase did not run cleanly; see $OUT/late.log" event late-end fi @@ -326,7 +325,7 @@ if (( ROLLOUT )); then # timer; a long bound here measures that wait instead of hiding it. event late-after-rollout-start log "late: ${LATE} members join behind the rollout" - join_cmd late-after-rollout "$LATE" 31000 --ready-timeout 600s + join_cmd late-after-rollout "$LATE" --ready-timeout 600s "${JOIN_CMD[@]}" 2> "$OUT/late-after-rollout.log" \ || log "late-after-rollout phase did not run cleanly; see $OUT/late-after-rollout.log" event late-after-rollout-end diff --git a/tests/scale/member-ladder.sh b/tests/scale/member-ladder.sh index 3b13057f..6d43a7db 100755 --- a/tests/scale/member-ladder.sh +++ b/tests/scale/member-ladder.sh @@ -237,5 +237,14 @@ for step in "${STEPS[@]}"; do done log "ladder complete: ${resident} members resident; holding until Ctrl-C or ${HOLD}" -# Stay until the first minion's hold ends or the operator interrupts. -while rssh "${STARTED_MINIONS[0]}" "pgrep -x agentmesh-bench >/dev/null"; do sleep 60; done +# Stay until the first minion's hold ends or the operator interrupts. The +# check tells "the run ended" (pgrep exits 1) from "ssh did not get through" +# (255): an IAP tunnel drops now and then, and a dropped session must not +# end a hold that carries thousands of members. +while :; do + rc=0 + rssh "${STARTED_MINIONS[0]}" "pgrep -x agentmesh-bench >/dev/null" || rc=$? + if [[ $rc -eq 1 ]]; then break; fi + if [[ $rc -ne 0 ]]; then log "could not reach ${STARTED_MINIONS[0]} to check the hold (ssh exit ${rc}); checking again"; fi + sleep 60 +done