feat(integrations): Google Cloud metrics and resource poller

.env.example

@@ -191,6 +191,12 @@ INGEST_REQUIRE_TLS=false
 # Logs need no configuration. For metrics, set the email of a Google service account Maple owns;
 # the setup script lets it impersonate a read-only metrics account in the customer's project.
 # MAPLE_GCP_SERVICE_ACCOUNT_EMAIL=
+# A key of that same account, for the metrics and resource poller (alerting Worker). The JSON
+# key file, base64-encoded on one line so it survives a Worker secret and an env file unchanged:
+#   base64 < key.json | tr -d '\n'
+# Unset: nothing is polled. The account needs no role of its own, but its project must have
+# the IAM Service Account Credentials API enabled.
+# MAPLE_GCP_SERVICE_ACCOUNT_KEY=
 
 # OpenTelemetry (API self-observability) — read by @maple-dev/effect-sdk
 # MAPLE_ENVIRONMENT=local                  # "local" = no export; any other value enables OTLP

apps/alerting/src/schedule.ts

@@ -23,7 +23,14 @@ export const ALERTING_SCHEDULE = {
 	// evaluates, so a one-minute bound would starve the drain on a slow minute.
 	"* * * * *": { lanes: ["alert", "error", "escalation"], deadline: "110 seconds" },
 	"*/5 * * * *": {
-		lanes: ["anomaly", "cloudflare_analytics", "planetscale", "railway_metrics", "pr_review_post_merge"],
+		lanes: [
+			"anomaly",
+			"cloudflare_analytics",
+			"gcp_metrics",
+			"planetscale",
+			"railway_metrics",
+			"pr_review_post_merge",
+		],
 		deadline: "270 seconds",
 	},
 	"*/15 * * * *": { lanes: ["digest"], deadline: "10 minutes" },

apps/alerting/src/scheduled.test.ts

@@ -6,7 +6,17 @@ import { scheduleFor } from "./schedule"
 import { layerTick, catchTickFailure, selectLaneProgram, type ScheduledTickPrograms } from "./scheduled"
 
 const cronCases = [
-	["*/5 * * * *", ["anomaly", "cloudflareAnalytics", "planetScale", "railwayMetrics", "prReviewPostMerge"]],
+	[
+		"*/5 * * * *",
+		[
+			"anomaly",
+			"cloudflareAnalytics",
+			"gcpMetrics",
+			"planetScale",
+			"railwayMetrics",
+			"prReviewPostMerge",
+		],
+	],
 	["*/15 * * * *", ["digest"]],
 	["0 * * * *", ["serviceMapRollup"]],
 	["* * * * *", ["alert", "error", "escalation", "fixVerification"]],
@@ -26,6 +36,7 @@ const recordingTicks = (
 		error: tick("error"),
 		escalation: tick("escalation"),
 		fixVerification: tick("fixVerification"),
+		gcpMetrics: tick("gcpMetrics"),
 		planetScale: tick("planetScale"),
 		prReviewPostMerge: tick("prReviewPostMerge"),
 		railwayMetrics: tick("railwayMetrics"),

apps/alerting/src/scheduled.ts

@@ -15,6 +15,7 @@ import { Env } from "@maple/backend/platform/Env"
 import { ErrorsService } from "@maple/backend/services/errors/ErrorsService"
 import { EscalationService } from "@maple/backend/services/alerts/EscalationService"
 import { FixVerificationTickService } from "@maple/backend/services/errors/FixVerificationTickService"
+import { GcpMetricsService } from "@maple/backend/services/integrations/GcpMetricsService"
 import { IncidentClassifier } from "@maple/backend/services/errors/IncidentClassifier"
 import { DatabasePgLive } from "@maple/backend/platform/DatabasePgLive"
 import { PullRequestLookupLive } from "@maple/backend/services/errors/pull-request-lookup-live"
@@ -49,6 +50,7 @@ export const layerTick = (
 		AlertsService.layer,
 		AnomalyDetectionService.layer,
 		CloudflareAnalyticsService.layer,
+		GcpMetricsService.layer,
 		PlanetScaleService.layer,
 		RailwayMetricsService.layer,
 		DigestService.layer,
@@ -251,6 +253,22 @@ const cloudflareAnalyticsTick = makeTick(
 	}),
 )
 
+const gcpMetricsTick = makeTick(
+	GcpMetricsService.use((gcp) => gcp.pollAll()),
+	"gcp_metrics",
+	(result) =>
+		result.polled + result.deferred > 0
+			? {
+					polled: result.polled,
+					deferred: result.deferred,
+					rowsIngested: result.rowsIngested,
+					failures: result.failures,
+					incompleteMetrics: result.incompleteMetrics,
+					calls: result.calls,
+				}
+			: undefined,
+)
+
 const planetScaleTick = makeTick(
 	PlanetScaleService.use((planetscale) => planetscale.pollAllOrgs()),
 	"planetscale",
@@ -303,6 +321,7 @@ export interface ScheduledTickPrograms<R = never> {
 	readonly error: Effect.Effect<void, never, R>
 	readonly escalation: Effect.Effect<void, never, R>
 	readonly fixVerification: Effect.Effect<void, never, R>
+	readonly gcpMetrics: Effect.Effect<void, never, R>
 	readonly planetScale: Effect.Effect<void, never, R>
 	readonly prReviewPostMerge: Effect.Effect<void, never, R>
 	readonly railwayMetrics: Effect.Effect<void, never, R>
@@ -320,6 +339,7 @@ const laneProgram = <R>(lane: TickLane, ticks: ScheduledTickPrograms<R>): Effect
 		Match.when("escalation", () => ticks.escalation),
 		Match.when("anomaly", () => ticks.anomaly),
 		Match.when("cloudflare_analytics", () => ticks.cloudflareAnalytics),
+		Match.when("gcp_metrics", () => ticks.gcpMetrics),
 		Match.when("planetscale", () => ticks.planetScale),
 		Match.when("railway_metrics", () => ticks.railwayMetrics),
 		Match.when("pr_review_post_merge", () => ticks.prReviewPostMerge),
@@ -369,6 +389,7 @@ type ScheduledServices =
 	| ErrorsService
 	| EscalationService
 	| FixVerificationTickService
+	| GcpMetricsService
 	| PlanetScaleService
 	| PrReviewPostMergeService
 	| RailwayMetricsService
@@ -382,6 +403,7 @@ export const scheduledTicks: ScheduledTickPrograms<ScheduledServices> = {
 	error: errorTick,
 	escalation: escalationTick,
 	fixVerification: fixVerificationTick,
+	gcpMetrics: gcpMetricsTick,
 	planetScale: planetScaleTick,
 	prReviewPostMerge: prReviewPostMergeTick,
 	railwayMetrics: railwayMetricsTick,

apps/alerting/src/worker.ts

@@ -67,6 +67,8 @@ const configuredEnv = (stage: MapleStage, region: MapleRegion, domains: MapleDom
 		apnsEnv,
 		cloudflareOAuthEnv,
 		planetScaleOAuthEnv,
+		// Sign-in for the Google Cloud metrics poller; unset leaves its tick a no-op.
+		optionalSecret("MAPLE_GCP_SERVICE_ACCOUNT_KEY"),
 		// Ticks post pull-request replies through the GitHub App; without it each one fails.
 		githubAppSourceEnv,
 		// Outbound config for `chat` destinations' connectors.

apps/api/src/routes/v2/integrations-gcp.http.test.ts

@@ -158,6 +158,10 @@ describe("v2 gcp integration over HTTP", () => {
 			applied_logs_enabled: null,
 			applied_metrics_enabled: null,
 			setup_reported_at: null,
+			last_metrics_received_at: null,
+			last_metrics_error: null,
+			discovered_project_count: 0,
+			last_resources_error: null,
 		})
 
 		// A project is its own host project, and forwards logs only unless asked otherwise.

apps/api/src/routes/v2/integrations-gcp.http.ts

@@ -34,6 +34,10 @@ const toV2Connector = (connector: GcpConnector): V2GcpConnector => ({
 	applied_logs_enabled: connector.appliedLogsEnabled,
 	applied_metrics_enabled: connector.appliedMetricsEnabled,
 	setup_reported_at: isoTimestampOrNull(connector.setupReportedAt),
+	last_metrics_received_at: isoTimestampOrNull(connector.lastMetricsReceivedAt),
+	last_metrics_error: connector.lastMetricsError,
+	discovered_project_count: connector.discoveredProjectCount,
+	last_resources_error: connector.lastResourcesError,
 })
 
 const auditMetadata = (connector: GcpConnector) => ({

packages/backend/src/platform/Env.ts

@@ -151,6 +151,11 @@ export interface EnvConfig {
 	 * metrics reader their setup script creates. Unset: the GCP integration is logs-only.
 	 */
 	readonly MAPLE_GCP_SERVICE_ACCOUNT_EMAIL: Option.Option<string>
+	/**
+	 * A key of that service account: its JSON key file, base64-encoded. The metrics and resource
+	 * poller signs in with it. Unset: nothing is polled.
+	 */
+	readonly MAPLE_GCP_SERVICE_ACCOUNT_KEY: Option.Option<Redacted.Redacted<string>>
 	readonly CLOUDFLARE_OAUTH_CLIENT_ID: Option.Option<string>
 	readonly CLOUDFLARE_OAUTH_CLIENT_SECRET: Option.Option<Redacted.Redacted<string>>
 	readonly CLOUDFLARE_OAUTH_SCOPES: string
@@ -260,6 +265,7 @@ const envConfig = Config.all({
 	GITHUB_APP_WEBHOOK_SECRET: optionalRedacted("GITHUB_APP_WEBHOOK_SECRET"),
 	GITHUB_API_BASE_URL: stringWithDefault("GITHUB_API_BASE_URL", "https://api.github.com"),
 	MAPLE_GCP_SERVICE_ACCOUNT_EMAIL: optionalString("MAPLE_GCP_SERVICE_ACCOUNT_EMAIL"),
+	MAPLE_GCP_SERVICE_ACCOUNT_KEY: optionalRedacted("MAPLE_GCP_SERVICE_ACCOUNT_KEY"),
 	CLOUDFLARE_OAUTH_CLIENT_ID: optionalString("CLOUDFLARE_OAUTH_CLIENT_ID"),
 	CLOUDFLARE_OAUTH_CLIENT_SECRET: optionalRedacted("CLOUDFLARE_OAUTH_CLIENT_SECRET"),
 	// Cloudflare OAuth scope ids are DOT-delimited (mirroring API-token permission names;

packages/backend/src/services/integrations/GcpConnectorService.test.ts

@@ -244,6 +244,97 @@ describe("GcpConnectorService", () => {
 		}).pipe(Effect.provide(makeLayer(testDb)))
 	})
 
+	it.effect("reports what the poller recorded and how many projects it found", () => {
+		const testDb = createTestDb(trackedDbs)
+		return Effect.gen(function* () {
+			const gcp = yield* GcpConnectorService
+			const connector = yield* gcp.create(orgId, userId, organizationScope)
+			assert.deepInclude(connector, {
+				lastMetricsReceivedAt: null,
+				lastMetricsError: null,
+				discoveredProjectCount: 0,
+				lastResourcesError: null,
+			})
+			const other = yield* gcp.create(orgId, userId, projectScope("acme-prod"))
+			// What the poller writes: its state on the connector, and the scope's inventory.
+			yield* Effect.promise(() =>
+				executeSql(
+					testDb,
+					`UPDATE gcp_connectors
+					 SET metrics_watermark_at = $1, last_metrics_received_at = $1, last_metrics_error = $2,
+					     resources_synced_at = $1, last_resources_error = $3
+					 WHERE id = $4`,
+					[
+						"2026-10-08T09:10:00.000Z",
+						"2 of 46 metric queries failed.",
+						"incomplete",
+						connector.id,
+					],
+				),
+			)
+			for (const [name, assetType] of [
+				[
+					"//cloudresourcemanager.googleapis.com/projects/111",
+					"cloudresourcemanager.googleapis.com/Project",
+				],
+				[
+					"//cloudresourcemanager.googleapis.com/projects/222",
+					"cloudresourcemanager.googleapis.com/Project",
+				],
+				[
+					"//run.googleapis.com/projects/shop/locations/eu/services/api",
+					"run.googleapis.com/Service",
+				],
+			]) {
+				yield* Effect.promise(() =>
+					executeSql(
+						testDb,
+						`INSERT INTO gcp_resources (connector_id, org_id, name, asset_type, project_id, labels, last_seen_at)
+						 VALUES ($1, $2, $3, $4, 'shop', '{}', now())`,
+						[connector.id, orgId, name, assetType],
+					),
+				)
+			}
+			const polled = {
+				...connector,
+				lastMetricsReceivedAt: Date.parse("2026-10-08T09:10:00.000Z"),
+				lastMetricsError: "2 of 46 metric queries failed.",
+				discoveredProjectCount: 2,
+				lastResourcesError: "incomplete",
+			}
+			// Both were created in the same test-clock instant, so their order is by id.
+			assert.sameDeepMembers([...(yield* gcp.status(orgId)).connectors], [polled, other])
+
+			// Switching metrics off leaves the poller's state as it is.
+			assert.deepStrictEqual(yield* gcp.update(orgId, connector.id, { metricsEnabled: false }), {
+				...polled,
+				metricsEnabled: false,
+			})
+			// Back on, the connection waits for its first metrics again; the poller continues
+			// from its watermark.
+			assert.deepStrictEqual(yield* gcp.update(orgId, connector.id, { metricsEnabled: true }), {
+				...polled,
+				lastMetricsReceivedAt: null,
+				lastMetricsError: null,
+			})
+			const kept = yield* Effect.promise(() =>
+				queryFirstRow<{ metrics_watermark_at: Date | null }>(
+					testDb,
+					"SELECT metrics_watermark_at FROM gcp_connectors WHERE id = $1",
+					[connector.id],
+				),
+			)
+			assert.strictEqual(kept?.metrics_watermark_at?.getTime(), Date.parse("2026-10-08T09:10:00.000Z"))
+
+			// The inventory goes with its connector.
+			yield* gcp.delete(orgId, connector.id)
+			const left = yield* Effect.promise(() =>
+				queryFirstRow<{ count: number }>(testDb, "SELECT count(*)::int AS count FROM gcp_resources"),
+			)
+			assert.strictEqual(left?.count, 0)
+		}).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT)))
+	})
+
 	it.effect("refuses to render a secret that was moved onto another connector's row", () => {
 		const testDb = createTestDb(trackedDbs)
 		return Effect.gen(function* () {

packages/backend/src/services/integrations/GcpConnectorService.ts

@@ -1,7 +1,8 @@
 import { randomBytes, randomUUID } from "node:crypto"
 import * as PG from "@maple-dev/effect-orm/postgres"
 import { hashIngestKey, parseIngestKeyLookupHmacKey } from "@maple/db"
-import { GcpConnectors, type GcpConnectorRow } from "@maple/db/tables"
+import { GcpConnectors, GcpResources, type GcpConnectorRow } from "@maple/db/tables"
+import { GCP_PROJECT_ASSET_TYPE } from "@maple/domain/gcp-metrics"
 import {
 	GcpMetricsUnavailableError,
 	GcpScopeAlreadyConnectedError,
@@ -43,6 +44,12 @@ export interface GcpConnector {
 	readonly appliedLogsEnabled: boolean | null
 	readonly appliedMetricsEnabled: boolean | null
 	readonly setupReportedAt: number | null
+	/** Last poll that read the scope's metrics. Null until the first one. */
+	readonly lastMetricsReceivedAt: number | null
+	readonly lastMetricsError: string | null
+	/** Projects the last inventory sync found in the scope. */
+	readonly discoveredProjectCount: number
+	readonly lastResourcesError: string | null
 }
 
 export type CreateGcpConnectorInput = Pick<
@@ -104,7 +111,7 @@ const toPersistenceError = makePersistenceErrorMapper(
 // Binds the ciphertext to its row, so a stored secret cannot be moved onto another connector.
 const secretAad = (connectorId: string) => Buffer.from(`gcp_connectors:v1:${connectorId}`, "utf8")
 
-const toConnector = (row: GcpConnectorRow): GcpConnector => ({
+const toConnector = (row: GcpConnectorRow, discoveredProjectCount = 0): GcpConnector => ({
 	id: row.id,
 	scopeType: row.scopeType,
 	scopeId: row.scopeId,
@@ -117,6 +124,10 @@ const toConnector = (row: GcpConnectorRow): GcpConnector => ({
 	appliedLogsEnabled: row.appliedLogsEnabled,
 	appliedMetricsEnabled: row.appliedMetricsEnabled,
 	setupReportedAt: row.setupReportedAt,
+	lastMetricsReceivedAt: row.lastMetricsReceivedAt,
+	lastMetricsError: row.lastMetricsError,
+	discoveredProjectCount,
+	lastResourcesError: row.lastResourcesError,
 })
 
 const notFound = () => new IntegrationsNotFoundError({ message: "No such Google Cloud connector." })
@@ -173,6 +184,20 @@ export class GcpConnectorService extends Context.Service<GcpConnectorService, Gc
 					),
 				)
 
+			/** Projects in each connector's inventory. One the poller has not synced yet is absent. */
+			const projectCounts = (connectorIds: ReadonlyArray<GcpConnectorId>) =>
+				dbExecute((db) =>
+					db.run(
+						PG.from(GcpResources)
+							.select(($) => ({ connectorId: $.connectorId, count: PG.count() }))
+							.where(($) => [
+								$.connectorId.in_(...connectorIds),
+								$.assetType.eq(GCP_PROJECT_ASSET_TYPE),
+							])
+							.groupBy("connectorId"),
+					),
+				).pipe(Effect.map((rows) => new Map(rows.map((row) => [row.connectorId, row.count]))))
+
 			const status = Effect.fn("GcpConnectorService.status")(function* (orgId: OrgId) {
 				yield* Effect.annotateCurrentSpan({ orgId })
 				const rows = yield* dbExecute((db) =>
@@ -183,9 +208,14 @@ export class GcpConnectorService extends Context.Service<GcpConnectorService, Gc
 							.orderBy(["createdAt", "asc"], ["id", "asc"]),
 					),
 				)
+				// No connectors, nothing to count.
+				const counts =
+					rows.length === 0
+						? new Map<GcpConnectorId, number>()
+						: yield* projectCounts(rows.map((row) => row.id))
 				return {
 					metricsAvailable: mapleServiceAccountEmail !== undefined,
-					connectors: rows.map(toConnector),
+					connectors: rows.map((row) => toConnector(row, counts.get(row.id))),
 				}
 			})
 
@@ -273,6 +303,10 @@ export class GcpConnectorService extends Context.Service<GcpConnectorService, Gc
 								...(flags.logsEnabled && !current.logsEnabled
 									? { lastReceivedAt: null, lastError: null }
 									: undefined),
+								// Only what the card shows: the poller's watermark and lease stay.
+								...(flags.metricsEnabled && !current.metricsEnabled
+									? { lastMetricsReceivedAt: null, lastMetricsError: null }
+									: undefined),
 							}
 							if (invalid === undefined) {
 								yield* db.run(
@@ -287,7 +321,8 @@ export class GcpConnectorService extends Context.Service<GcpConnectorService, Gc
 				)
 				if (outcome === undefined) return yield* notFound()
 				if (outcome.invalid !== undefined) return yield* Effect.fail(outcome.invalid)
-				return toConnector(outcome.row)
+				const { row } = outcome
+				return toConnector(row, (yield* projectCounts([row.id])).get(row.id))
 			})
 
 			const scripts = Effect.fn("GcpConnectorService.scripts")(function* (