From 6f80d17d0fc31073c7fb8b9c5d639da7ef0d26d7 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Thu, 15 Oct 2026 10:08:57 +0000 Subject: [PATCH 1/4] controlplane: a bootstrap member that fell silent is forgotten; ephemeral tokens A bootstrap enrollment has no session, so the node GC, which deletes rows whose session lapsed, never matched one: every member that joined with a token and left stayed enrolled for good. The ladder tests alone left a hundred thousand such rows on the testnet, and the control plane's full scans of that table were what stalled it under load. The GC now also deletes an unbanned bootstrap row that has not refreshed for --node-retention: enrolled_at is rewritten by every refresh, so it already is the last time the member was heard from. A member that keeps refreshing keeps its row; one that fell silent for a month is forgotten, and enrolls again if it comes back, as a self-hosted CI runner is removed after a fortnight without connecting and re-registers. Nothing a running member can observe changes: no session, no refusal, no new state. A bootstrap token can be ephemeral (BootstrapTokenCreateRequest.ephemeral, a checkbox in the console). The members it enrolls carry the flag and are forgotten after --ephemeral-node-retention (an hour) without a refresh: a fleet in a scale test, a probe, a CI job. The ladder and journey harnesses mint ephemeral tokens, so a scale run leaves the table as it found it within the hour. agentmesh_control_plane_nodes_deleted_total gains a reason label: session, silent, ephemeral. Migration 20 adds the ephemeral column to tokens and nodes. agentmesh-one exposes the retention as --control-plane-ephemeral-node-retention. --- api/agentmesh.pb.go | 54 ++++++-- api/agentmesh.proto | 10 ++ cmd/agentmesh-control-plane/main.go | 100 +++++++------- cmd/agentmesh-one/main.go | 3 +- internal/console/public/app.js | 6 + internal/console/public/index.html | 7 + internal/controlplane/config.go | 20 ++- internal/controlplane/node_gc.go | 54 ++++++-- internal/controlplane/node_gc_test.go | 75 ++++++++++- internal/controlplane/server.go | 4 + internal/controlplane/server_test.go | 14 +- internal/controlplane/views.go | 2 + internal/standalone/standalone.go | 45 ++++--- internal/storage/round_trip_test.go | 52 ++++++++ internal/storage/sql_store.go | 68 ++++++++-- internal/storage/storage.go | 22 +++- sdk/js/src/gen/agentmesh_pb.ts | 26 +++- .../src/agent_mesh/_proto/agentmesh_pb2.py | 124 +++++++++--------- .../src/agent_mesh/_proto/agentmesh_pb2.pyi | 18 ++- site/content/docs/concepts/identity.md | 8 +- .../docs/guides/headless-enrollment.md | 4 + site/content/docs/reference/agentmesh-one.md | 1 + site/content/docs/reference/control-plane.md | 16 ++- 23 files changed, 542 insertions(+), 191 deletions(-) diff --git a/api/agentmesh.pb.go b/api/agentmesh.pb.go index d06b591a..7f86a0b1 100644 --- a/api/agentmesh.pb.go +++ b/api/agentmesh.pb.go @@ -3984,8 +3984,14 @@ type BootstrapTokenCreateRequest struct { // because a node that can always recover holds a credential that never // expires. AutonomousRecovery bool `protobuf:"varint,6,opt,name=autonomous_recovery,json=autonomousRecovery,proto3" json:"autonomous_recovery,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Copied onto every node the token enrolls: such a node is forgotten + // soon after it falls silent (the control plane's + // --ephemeral-node-retention, an hour by default) instead of being kept + // for --node-retention. For members that come and go: a fleet in a scale + // test, a probe, a CI job. + Ephemeral bool `protobuf:"varint,7,opt,name=ephemeral,proto3" json:"ephemeral,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BootstrapTokenCreateRequest) Reset() { @@ -4060,6 +4066,13 @@ func (x *BootstrapTokenCreateRequest) GetAutonomousRecovery() bool { return false } +func (x *BootstrapTokenCreateRequest) GetEphemeral() bool { + if x != nil { + return x.Ephemeral + } + return false +} + // BootstrapTokenCreateResponse is returned (201) when a token is minted. // token is the plaintext and is shown exactly once; the control plane keeps // only its hash, which is also the id. @@ -4154,6 +4167,7 @@ type BootstrapToken struct { // or exhausted usages. Unset means never revoked. RevokeTime *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=revoke_time,json=revokeTime,proto3" json:"revoke_time,omitempty"` AutonomousRecovery bool `protobuf:"varint,10,opt,name=autonomous_recovery,json=autonomousRecovery,proto3" json:"autonomous_recovery,omitempty"` + Ephemeral bool `protobuf:"varint,11,opt,name=ephemeral,proto3" json:"ephemeral,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4258,6 +4272,13 @@ func (x *BootstrapToken) GetAutonomousRecovery() bool { return false } +func (x *BootstrapToken) GetEphemeral() bool { + if x != nil { + return x.Ephemeral + } + return false +} + type BootstrapTokenListResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Tokens []*BootstrapToken `protobuf:"bytes,1,rep,name=tokens,proto3" json:"tokens,omitempty"` @@ -4546,8 +4567,11 @@ type EnrolledNode struct { ExpireTime *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=expire_time,json=expireTime,proto3" json:"expire_time,omitempty"` Banned bool `protobuf:"varint,9,opt,name=banned,proto3" json:"banned,omitempty"` AutonomousRecovery bool `protobuf:"varint,10,opt,name=autonomous_recovery,json=autonomousRecovery,proto3" json:"autonomous_recovery,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The node was enrolled by an ephemeral token and is forgotten soon after + // it falls silent. + Ephemeral bool `protobuf:"varint,11,opt,name=ephemeral,proto3" json:"ephemeral,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EnrolledNode) Reset() { @@ -4650,6 +4674,13 @@ func (x *EnrolledNode) GetAutonomousRecovery() bool { return false } +func (x *EnrolledNode) GetEphemeral() bool { + if x != nil { + return x.Ephemeral + } + return false +} + // RouterLease is a router's current registration with the control plane. type RouterLease struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -6328,7 +6359,7 @@ const file_api_agentmesh_proto_rawDesc = "" + "\apeer_id\x18\x01 \x01(\tR\x06peerId\"E\n" + "\x13TokenRevokeResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12\x14\n" + - "\x05error\x18\x02 \x01(\tR\x05error\"\xdb\x01\n" + + "\x05error\x18\x02 \x01(\tR\x05error\"\xf9\x01\n" + "\x1bBootstrapTokenCreateRequest\x12\x12\n" + "\x04role\x18\x01 \x01(\tR\x04role\x12\x19\n" + "\bowner_id\x18\x02 \x01(\tR\aownerId\x12\x1b\n" + @@ -6336,14 +6367,15 @@ const file_api_agentmesh_proto_rawDesc = "" + "\n" + "max_usages\x18\x04 \x01(\x05R\tmaxUsages\x12 \n" + "\vdescription\x18\x05 \x01(\tR\vdescription\x12/\n" + - "\x13autonomous_recovery\x18\x06 \x01(\bR\x12autonomousRecovery\"\xb0\x01\n" + + "\x13autonomous_recovery\x18\x06 \x01(\bR\x12autonomousRecovery\x12\x1c\n" + + "\tephemeral\x18\a \x01(\bR\tephemeral\"\xb0\x01\n" + "\x1cBootstrapTokenCreateResponse\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + "\x05token\x18\x02 \x01(\tR\x05token\x12\x12\n" + "\x04role\x18\x03 \x01(\tR\x04role\x12\x19\n" + "\bowner_id\x18\x04 \x01(\tR\aownerId\x12;\n" + "\vexpire_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + - "expireTime\"\x9b\x03\n" + + "expireTime\"\xb9\x03\n" + "\x0eBootstrapToken\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + "\x04role\x18\x02 \x01(\tR\x04role\x12\x19\n" + @@ -6359,7 +6391,8 @@ const file_api_agentmesh_proto_rawDesc = "" + "\vrevoke_time\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\n" + "revokeTime\x12/\n" + "\x13autonomous_recovery\x18\n" + - " \x01(\bR\x12autonomousRecovery\"R\n" + + " \x01(\bR\x12autonomousRecovery\x12\x1c\n" + + "\tephemeral\x18\v \x01(\bR\tephemeral\"R\n" + "\x1aBootstrapTokenListResponse\x124\n" + "\x06tokens\x18\x01 \x03(\v2\x1c.agentmesh.v1.BootstrapTokenR\x06tokens\"\xac\x03\n" + "\x11EnrollmentRequest\x12\x0e\n" + @@ -6384,7 +6417,7 @@ const file_api_agentmesh_proto_rawDesc = "" + "\x05email\x18\x03 \x01(\tR\x05email\x12\x12\n" + "\x04role\x18\x04 \x01(\tR\x04role\x12;\n" + "\vcreate_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + - "createTime\"\xde\x03\n" + + "createTime\"\xfc\x03\n" + "\fEnrolledNode\x12\x17\n" + "\apeer_id\x18\x01 \x01(\tR\x06peerId\x12\x12\n" + "\x04role\x18\x02 \x01(\tR\x04role\x12'\n" + @@ -6399,7 +6432,8 @@ const file_api_agentmesh_proto_rawDesc = "" + "expireTime\x12\x16\n" + "\x06banned\x18\t \x01(\bR\x06banned\x12/\n" + "\x13autonomous_recovery\x18\n" + - " \x01(\bR\x12autonomousRecovery\x1a9\n" + + " \x01(\bR\x12autonomousRecovery\x12\x1c\n" + + "\tephemeral\x18\v \x01(\bR\tephemeral\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8d\x02\n" + diff --git a/api/agentmesh.proto b/api/agentmesh.proto index 0d76de6a..ac0e8d05 100644 --- a/api/agentmesh.proto +++ b/api/agentmesh.proto @@ -667,6 +667,12 @@ message BootstrapTokenCreateRequest { // because a node that can always recover holds a credential that never // expires. bool autonomous_recovery = 6; + // Copied onto every node the token enrolls: such a node is forgotten + // soon after it falls silent (the control plane's + // --ephemeral-node-retention, an hour by default) instead of being kept + // for --node-retention. For members that come and go: a fleet in a scale + // test, a probe, a CI job. + bool ephemeral = 7; } // BootstrapTokenCreateResponse is returned (201) when a token is minted. @@ -694,6 +700,7 @@ message BootstrapToken { // or exhausted usages. Unset means never revoked. google.protobuf.Timestamp revoke_time = 9; bool autonomous_recovery = 10; + bool ephemeral = 11; } message BootstrapTokenListResponse { @@ -746,6 +753,9 @@ message EnrolledNode { google.protobuf.Timestamp expire_time = 8; bool banned = 9; bool autonomous_recovery = 10; + // The node was enrolled by an ephemeral token and is forgotten soon after + // it falls silent. + bool ephemeral = 11; } // RouterLease is a router's current registration with the control plane. diff --git a/cmd/agentmesh-control-plane/main.go b/cmd/agentmesh-control-plane/main.go index ebf3f158..fcec9af8 100644 --- a/cmd/agentmesh-control-plane/main.go +++ b/cmd/agentmesh-control-plane/main.go @@ -36,31 +36,32 @@ import ( ) var ( - bindAddress string - pprofAddr string - dbDriver string - dbDSN string - dbDSNPath string - oidcIssuer string - workloadIssuer string - oidcClientID string - allowedAudiencesFlag string - keyRotationInterval time.Duration - keyGracePeriod time.Duration - leaseDuration time.Duration - dhtReplication int - dhtReprovideInterval time.Duration - biscuitTTL time.Duration - oidcSessionTTL time.Duration - workloadSessionTTL time.Duration - nodeRetention time.Duration - meshReconnectInterval time.Duration - adminTokenPath string - stsIssuerURL string - insecureSkipTLSVerify bool - logLevel string - logFormat string - autoApproveEnrollment bool + bindAddress string + pprofAddr string + dbDriver string + dbDSN string + dbDSNPath string + oidcIssuer string + workloadIssuer string + oidcClientID string + allowedAudiencesFlag string + keyRotationInterval time.Duration + keyGracePeriod time.Duration + leaseDuration time.Duration + dhtReplication int + dhtReprovideInterval time.Duration + biscuitTTL time.Duration + oidcSessionTTL time.Duration + workloadSessionTTL time.Duration + nodeRetention time.Duration + ephemeralNodeRetention time.Duration + meshReconnectInterval time.Duration + adminTokenPath string + stsIssuerURL string + insecureSkipTLSVerify bool + logLevel string + logFormat string + autoApproveEnrollment bool ) var logger = golog.Logger("agentmesh-control-plane-cli") @@ -102,6 +103,9 @@ func main() { if nodeRetention <= 0 { logger.Fatalf("--node-retention must be positive, got %s: node records are never kept forever", nodeRetention) } + if ephemeralNodeRetention <= 0 { + logger.Fatalf("--ephemeral-node-retention must be positive, got %s", ephemeralNodeRetention) + } var auds []string for _, aud := range strings.Split(allowedAudiencesFlag, ",") { @@ -126,27 +130,28 @@ func main() { }() opts := controlplane.Options{ - ListenAddr: bindAddress, - DriverName: dbDriver, - DataSourceName: dbDSN, - OIDCIssuer: oidcIssuer, - WorkloadIssuer: workloadIssuer, - OIDCClientID: oidcClientID, - AllowedAudiences: auds, - LeaseDuration: leaseDuration, - KeyRotationInterval: keyRotationInterval, - KeyGracePeriod: keyGracePeriod, - DHTReplication: dhtReplication, - DHTReprovideInterval: dhtReprovideInterval, - InsecureSkipTLSVerify: insecureSkipTLSVerify, - BiscuitTimeout: 10 * time.Second, - BiscuitTTL: biscuitTTL, - OIDCSessionTTL: oidcSessionTTL, - WorkloadSessionTTL: workloadSessionTTL, - NodeRetention: nodeRetention, - AdminToken: adminToken, - AutoApproveEnrollment: autoApproveEnrollment, - STSIssuerURL: stsIssuerURL, + ListenAddr: bindAddress, + DriverName: dbDriver, + DataSourceName: dbDSN, + OIDCIssuer: oidcIssuer, + WorkloadIssuer: workloadIssuer, + OIDCClientID: oidcClientID, + AllowedAudiences: auds, + LeaseDuration: leaseDuration, + KeyRotationInterval: keyRotationInterval, + KeyGracePeriod: keyGracePeriod, + DHTReplication: dhtReplication, + DHTReprovideInterval: dhtReprovideInterval, + InsecureSkipTLSVerify: insecureSkipTLSVerify, + BiscuitTimeout: 10 * time.Second, + BiscuitTTL: biscuitTTL, + OIDCSessionTTL: oidcSessionTTL, + WorkloadSessionTTL: workloadSessionTTL, + NodeRetention: nodeRetention, + EphemeralNodeRetention: ephemeralNodeRetention, + AdminToken: adminToken, + AutoApproveEnrollment: autoApproveEnrollment, + STSIssuerURL: stsIssuerURL, } srv, err := controlplane.NewServer(opts, store) @@ -206,7 +211,8 @@ func main() { rootCmd.Flags().DurationVar(&biscuitTTL, "biscuit-ttl", api.BiscuitTokenTTL, "Lifespan minted into every issued Biscuit's expiration fact. Capped to the OIDC token's own expiry when shorter.") rootCmd.Flags().DurationVar(&oidcSessionTTL, "oidc-session-ttl", api.OIDCSessionTTL, "How long an OIDC enrollment stays refreshable before the identity must re-authenticate with the OIDC provider. Shorter values keep the provider authoritative for offboarding at the cost of more frequent interactive re-enrollment.") rootCmd.Flags().DurationVar(&workloadSessionTTL, "workload-session-ttl", controlplane.DefaultWorkloadSessionTTL, "How long a workload-issuer enrollment stays refreshable without presenting a fresh platform JWT on /refresh.") - rootCmd.Flags().DurationVar(&nodeRetention, "node-retention", controlplane.DefaultNodeRetention, "How long an enrolled node's record is kept after its session expired before it is deleted. Banned nodes are always kept. Must be positive: records are never kept forever.") + rootCmd.Flags().DurationVar(&nodeRetention, "node-retention", controlplane.DefaultNodeRetention, "How long an enrolled node's record is kept after its session expired, or after its last refresh for a node enrolled with a bootstrap token, before it is deleted. Banned nodes are always kept. Must be positive: records are never kept forever.") + rootCmd.Flags().DurationVar(&ephemeralNodeRetention, "ephemeral-node-retention", controlplane.DefaultEphemeralNodeRetention, "--node-retention for a node enrolled with an ephemeral bootstrap token: how long after its last refresh its record is kept. For members that come and go: a fleet in a scale test, a probe, a CI job.") rootCmd.Flags().DurationVar(&meshReconnectInterval, "mesh-reconnect-interval", controlplane.DefaultMeshReconnectInterval, "How often the event publisher re-reads the router leases and dials any router it is not connected to.") rootCmd.Flags().StringVar(&adminTokenPath, "admin-token-path", "", "Path to file containing the token for authenticating policy REST API requests (or env AGENTMESH_ADMIN_TOKEN)") rootCmd.Flags().StringVar(&stsIssuerURL, "sts-issuer-url", "", "Canonical external URL of this control plane for OIDC/STS issuer and discovery metadata (or env AGENTMESH_STS_ISSUER_URL)") diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.go index 9fade107..a48728f6 100644 --- a/cmd/agentmesh-one/main.go +++ b/cmd/agentmesh-one/main.go @@ -231,7 +231,8 @@ func main() { rootCmd.Flags().DurationVar(&cpTunables.KeyGracePeriod, "control-plane-key-grace-period", 0, "How long rotated-out keys stay valid for verification (0 keeps the component default)") rootCmd.Flags().DurationVar(&cpTunables.BiscuitTTL, "control-plane-biscuit-ttl", 0, "Lifespan minted into issued biscuits (0 keeps the component default)") rootCmd.Flags().DurationVar(&cpTunables.WorkloadSessionTTL, "control-plane-workload-session-ttl", 0, "How long a workload-issuer enrollment stays refreshable without presenting a fresh platform JWT on /refresh (0 keeps the component default)") - rootCmd.Flags().DurationVar(&cpTunables.NodeRetention, "control-plane-node-retention", 0, "How long a node record is kept after its session expired before it is deleted (0 keeps the component default)") + rootCmd.Flags().DurationVar(&cpTunables.NodeRetention, "control-plane-node-retention", 0, "How long a node record is kept after its session expired, or after its last refresh for a bootstrap enrollment, before it is deleted (0 keeps the component default)") + rootCmd.Flags().DurationVar(&cpTunables.EphemeralNodeRetention, "control-plane-ephemeral-node-retention", 0, "How long a node enrolled with an ephemeral bootstrap token is kept after its last refresh (0 keeps the component default)") rootCmd.Flags().BoolVar(&cpTunables.ManualEnrollment, "control-plane-manual-enrollment", false, "Queue bootstrap enrollments for admin approval instead of auto-approving") // Embedded router tunables. diff --git a/internal/console/public/app.js b/internal/console/public/app.js index b1bd4178..2216d3e7 100644 --- a/internal/console/public/app.js +++ b/internal/console/public/app.js @@ -779,6 +779,7 @@ function renderBootstrapTokensTable(tokens) { ? `Revoked` : `Active`; const recovery = token.autonomous_recovery ? 'Autonomous' : 'Manual'; + const members = token.ephemeral ? 'Ephemeral' : 'Regular'; const action = revoked ? '-' : ``; @@ -789,6 +790,7 @@ function renderBootstrapTokensTable(tokens) { ${escapeHTML(token.owner_id || '-')} ${escapeHTML(String(token.usages_count || 0))} / ${escapeHTML(String(token.max_usages || 0))} ${escapeHTML(expiresAt)} + ${members} ${recovery} ${status} ${action} @@ -810,6 +812,7 @@ window.generateBootstrapToken = async function() { const ttl_hours = parseInt(document.getElementById('token-ttl').value, 10) || 24; const description = document.getElementById('token-desc').value; const autonomous_recovery = document.getElementById('token-recovery').checked; + const ephemeral = document.getElementById('token-ephemeral').checked; const payload = { role, @@ -824,6 +827,9 @@ window.generateBootstrapToken = async function() { if (autonomous_recovery) { payload.autonomous_recovery = true; } + if (ephemeral) { + payload.ephemeral = true; + } try { const res = await actionRequest('api/user/bootstrap-tokens', 'POST', payload); diff --git a/internal/console/public/index.html b/internal/console/public/index.html index 90683cee..0dc61628 100644 --- a/internal/console/public/index.html +++ b/internal/console/public/index.html @@ -355,6 +355,12 @@

Generate Bootstrap Token

+
+ + Nodes enrolled with this token are forgotten an hour after they stop refreshing, instead of being kept for the node retention. For members that come and go: a fleet in a scale test, a probe, a CI job. +