From 62a3bb850a78250bd1a2fab8bccfdc142f53be67 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 01:06:32 +0200 Subject: [PATCH 1/8] feat(gcp): query Google Cloud metrics live with PromQL --- apps/api/src/resources/env.ts | 2 + .../src/routes/internal/integrations.http.ts | 22 + apps/api/src/runtime/service-graph.ts | 2 + .../integrations/GcpMetricsService.ts | 51 +- .../services/integrations/GcpPromQlService.ts | 463 ++++++++++++++++++ .../src/services/integrations/gcp/api.ts | 153 +++++- .../src/services/integrations/gcp/explain.ts | 56 +++ .../anticipated-error-identifiers.ts | 4 + packages/domain/src/http/gcp-promql.ts | 173 +++++++ packages/domain/src/http/index.ts | 1 + packages/domain/src/http/integrations.ts | 66 +++ 11 files changed, 941 insertions(+), 52 deletions(-) create mode 100644 packages/backend/src/services/integrations/GcpPromQlService.ts create mode 100644 packages/backend/src/services/integrations/gcp/explain.ts create mode 100644 packages/domain/src/http/gcp-promql.ts diff --git a/apps/api/src/resources/env.ts b/apps/api/src/resources/env.ts index 46814eecfb..202cef5f1d 100644 --- a/apps/api/src/resources/env.ts +++ b/apps/api/src/resources/env.ts @@ -80,4 +80,6 @@ export const apiConfiguredEnv = (stage: MapleStage, region: MapleRegion, domains cloudflareOAuthEnv, planetScaleOAuthEnv, optionalPlain("MAPLE_GCP_SERVICE_ACCOUNT_EMAIL"), + // Sign-in for live Google Cloud PromQL queries; unset answers them as unavailable. + optionalSecret("MAPLE_GCP_SERVICE_ACCOUNT_KEY"), ) diff --git a/apps/api/src/routes/internal/integrations.http.ts b/apps/api/src/routes/internal/integrations.http.ts index 629fd62ccf..c6b8d52fcb 100644 --- a/apps/api/src/routes/internal/integrations.http.ts +++ b/apps/api/src/routes/internal/integrations.http.ts @@ -55,6 +55,7 @@ import { type TopTrafficGroupDefinition, } from "@maple/backend/services/integrations/cloudflare-analytics/queries" import { GcpConnectorService } from "@maple/backend/services/integrations/GcpConnectorService" +import { GcpPromQlService } from "@maple/backend/services/integrations/GcpPromQlService" import { RailwayMetricsService } from "@maple/backend/services/integrations/RailwayMetricsService" import { GithubConnectService } from "@maple/backend/services/integrations/vcs/vendor/github/GithubConnectService" import { VcsCommitService } from "@maple/backend/services/integrations/vcs/VcsCommitService" @@ -120,6 +121,7 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte const cloudflareAnalytics = yield* CloudflareAnalyticsService const railway = yield* RailwayMetricsService const gcp = yield* GcpConnectorService + const gcpPromQl = yield* GcpPromQlService const edgeCache = yield* EdgeCacheService const env = yield* Env @@ -472,6 +474,26 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte return new GcpResourcesResponse(yield* gcp.resources(tenant.orgId, query)) }), ) + .handle("gcpPromQlQueryRange", ({ payload }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => + gcpPromQl.queryRange(tenant.orgId, payload), + ), + ) + .handle("gcpPromQlQuery", ({ payload }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => + gcpPromQl.queryInstant(tenant.orgId, payload), + ), + ) + .handle("gcpMetricSearch", ({ query }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => + gcpPromQl.searchMetrics(tenant.orgId, query), + ), + ) + .handle("gcpMetricDescribe", ({ query }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => + gcpPromQl.describeMetric(tenant.orgId, query), + ), + ) .handle("githubStatus", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index e9559c051b..9a8442ce8e 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -37,6 +37,7 @@ import { InvestigationService } from "@maple/backend/services/errors/Investigati import { RecommendationIssueService } from "@maple/backend/services/errors/RecommendationIssueService" import { CloudflareAnalyticsService } from "@maple/backend/services/integrations/CloudflareAnalyticsService" import { GcpConnectorService } from "@maple/backend/services/integrations/GcpConnectorService" +import { GcpPromQlService } from "@maple/backend/services/integrations/GcpPromQlService" import { PlanetScaleConnectionService } from "@maple/backend/services/integrations/PlanetScaleConnectionService" import { PlanetScaleDiscoveryService } from "@maple/backend/services/integrations/PlanetScaleDiscoveryService" import { PlanetScaleService } from "@maple/backend/services/integrations/PlanetScaleService" @@ -115,6 +116,7 @@ export const HttpServicesLive = Layer.mergeAll( CloudflareAnalyticsService.layer, RailwayMetricsService.layer, GcpConnectorService.layer, + GcpPromQlService.layer, OrgOpenRouterKeyService.layer, AuditLogService.layer, WarehouseQueryService.layer, diff --git a/packages/backend/src/services/integrations/GcpMetricsService.ts b/packages/backend/src/services/integrations/GcpMetricsService.ts index 7db7f3df1f..fa1269251c 100644 --- a/packages/backend/src/services/integrations/GcpMetricsService.ts +++ b/packages/backend/src/services/integrations/GcpMetricsService.ts @@ -33,6 +33,7 @@ import { type GcpReader, type GcpTimeSeriesQuery, } from "./gcp/api" +import { describeGcpApiError } from "./gcp/explain" import { mapGcpResource, mapGcpTimeSeries, type GcpMetricRows } from "./gcp/mapping" const MINUTE_MS = 60_000 @@ -79,50 +80,6 @@ const whenTooLarge = (connector: GcpConnectorRow) => ? "write to support@maple.dev" : "connect its folders or projects separately" -/** Why a failed call failed and what to do about it, in the customer's terms. */ -const explainApiError = ( - error: GcpApiError, - connector: GcpConnectorRow, - reading: "metrics" | "resources", -) => { - // The host project is the one Google checks for billing, enabled APIs and quota. A project - // scope is its own host, and reads as "the project". - const host = `${connector.scopeType === "project" ? "project" : "host project"} ${connector.projectId}` - const scope = `${connector.scopeType} ${connector.scopeId}` - if (error.reason === "BILLING_DISABLED") { - return `The ${host} has no active billing account, and ${error.api} only answers for projects that have one. Link one: https://console.cloud.google.com/billing/linkedaccount?project=${connector.projectId} Maple retries every 5 minutes.` - } - if (error.reason === "SERVICE_DISABLED") { - return `The ${error.api} API is switched off in the ${host}. Run the setup script again: it switches the API on.` - } - if (error.kind === "denied" || error.kind === "not_found") { - if (error.api === "IAM Credentials") { - // A connection that has been read before lost its account or its grant. - if (connector.lastMetricsReceivedAt !== null) { - return "Maple can no longer sign in as this connection's read-only service account. The account was deleted, or the grant to Maple was removed or is blocked by an organization policy. Run the setup script again: it restores both, or says what blocks it." - } - // Shown both before the script ever ran and for the minute or two after a run in - // which Google does not accept the new grant yet, so it has to read right in both. - return "Maple can't sign in as this connection's read-only service account yet. After a setup run Google needs a few minutes to accept the new grant, and Maple retries every 5 minutes. If this stays, the account does not exist, or the grant to Maple is missing or blocked by an organization policy: run the setup script and read its last lines." - } - return reading === "metrics" - ? `Google denied Maple's read of ${scope}: the read-only roles are missing. Run the setup script again: it grants them. A new grant can take a few minutes.` - : `Google denied the resource listing for ${scope}. Run the setup script again: it grants Cloud Asset Viewer.` - } - // Signing in is counted against Maple's own project, not the host project. - if (error.kind === "rate_limited" && error.api !== "IAM Credentials") { - return `Google rate-limited the ${error.api} API for the ${host}. ${RETRIES} If this repeats, raise that API's quota on the project.` - } - return `Maple's request to Google's ${error.api} API failed. ${RETRIES} If this repeats, write to support@maple.dev.` -} - -/** - * What the connection shows for a failed call: one sentence of cause, one of what to do, and - * Google's status at the end. - */ -const describeApiError = (error: GcpApiError, connector: GcpConnectorRow, reading: "metrics" | "resources") => - `${explainApiError(error, connector, reading)} (${error.message})` - const toPersistenceError = makePersistenceErrorMapper( IntegrationsPersistenceError, "Google Cloud metrics database error", @@ -359,7 +316,7 @@ export class GcpMetricsService extends Context.Service Effect.succeed({ - lastResourcesError: describeApiError(error, connector, "resources"), + lastResourcesError: describeGcpApiError(error, connector), }), "@maple/http/errors/IntegrationsPersistenceError": () => Effect.succeed({ @@ -478,7 +435,7 @@ export class GcpMetricsService extends Context.Service - recordFailure(connector, describeApiError(error, connector, "metrics")), + recordFailure(connector, describeGcpApiError(error, connector)), "@maple/api/integrations/GcpMetricsIngestError": (error) => error.status === 402 ? recordFailure( diff --git a/packages/backend/src/services/integrations/GcpPromQlService.ts b/packages/backend/src/services/integrations/GcpPromQlService.ts new file mode 100644 index 0000000000..8aed07bafd --- /dev/null +++ b/packages/backend/src/services/integrations/GcpPromQlService.ts @@ -0,0 +1,463 @@ +/** + * Live PromQL over the Google Cloud projects an organization has connected, through Google's + * Prometheus-compatible API. Every connection is a target, whatever its metrics switch: a project + * scope is its project, a folder or organization the active projects Cloud Asset finds under it. + * Each project is read with its connection's read-only account, one call per project, and what + * fails is reported beside what answered. + */ +import * as PG from "@maple-dev/effect-orm/postgres" +import { EdgeCacheService } from "@maple/cache" +import { GcpConnectors, type GcpConnectorRow } from "@maple/db/tables" +import { GCP_PROJECT_ASSET_TYPE } from "@maple/domain/gcp-metrics" +import { + GCP_PROMQL_MAX_PROJECTS, + GcpMetricDescriptor, + GcpMetricSearchResponse, + GcpPromQlFailedError, + type GcpPromQlFailure, + type GcpPromQlInstantRequest, + type GcpPromQlInvalidQueryError, + type GcpPromQlRangeRequest, + GcpPromQlResult, + GcpPromQlTooManyProjectsError, + gcpPromQlTimeMs, + GcpPromQlUnavailableError, + IntegrationsNotConnectedError, + IntegrationsNotFoundError, + IntegrationsPersistenceError, + IntegrationsValidationError, +} from "@maple/domain/http" +import { GcpProjectId, type OrgId } from "@maple/domain/primitives" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { Env } from "@maple/backend/platform/Env" +import { + fetchMapleAccessToken, + getMetricDescriptor, + type GcpApiError, + type GcpPromQlParams, + type GcpReader, + importServiceAccountKey, + impersonateReader, + listPromQlMetricNames, + promQlQuery, + searchResourcesPage, +} from "./gcp/api" +import { describeGcpApiError } from "./gcp/explain" +import { mapGcpResource } from "./gcp/mapping" + +/** Google's tokens live 600 s; one is reused until a minute before that. */ +const TOKEN_REUSE_MS = 9 * 60_000 +const CONCURRENCY = 4 +/** Projects a folder or organization search reads, 500 a page. */ +const MAX_PROJECT_PAGES = 4 +const HOUR_SECONDS = 3_600 +/** A window ending this close to now still gains points: its result is kept a minute, else ten. */ +const FRESH_MS = 5 * 60_000 +const DISTRIBUTION_SUFFIX = /_(bucket|count|sum)$/u + +const isProjectId = Schema.is(GcpProjectId) + +type Connector = Pick + +const scopeName = (connector: Connector) => `${connector.scopeType}s/${connector.scopeId}` + +export type GcpPromQlError = + | GcpPromQlUnavailableError + | IntegrationsNotConnectedError + | IntegrationsValidationError + | GcpPromQlInvalidQueryError + | GcpPromQlTooManyProjectsError + | GcpPromQlFailedError + | IntegrationsPersistenceError + +export interface GcpPromQlServiceApi { + readonly queryRange: ( + orgId: OrgId, + request: GcpPromQlRangeRequest, + ) => Effect.Effect + readonly queryInstant: ( + orgId: OrgId, + request: GcpPromQlInstantRequest, + ) => Effect.Effect + /** Metric names containing `search`, from one project's metric list. */ + readonly searchMetrics: ( + orgId: OrgId, + request: { + readonly search: string + readonly limit?: number | undefined + readonly project?: GcpProjectId | undefined + }, + ) => Effect.Effect + /** A metric type's descriptor. A distribution's `_bucket`, `_count` or `_sum` name finds it too. */ + readonly describeMetric: ( + orgId: OrgId, + request: { readonly type: string; readonly project?: GcpProjectId | undefined }, + ) => Effect.Effect +} + +const toPersistenceError = makePersistenceErrorMapper( + IntegrationsPersistenceError, + "Google Cloud connector database error", +) + +const notReadable = (projects: ReadonlyArray) => + new IntegrationsValidationError({ + message: `Not among the projects this organization's Google Cloud connections can read: ${projects.join(", ")}`, + }) + +const nothingRead = (failures: ReadonlyArray) => + new GcpPromQlFailedError({ + message: + failures.length === 0 + ? "The Google Cloud connections hold no project to read." + : `Google Cloud could not be read. ${failures.map((failure) => `${failure.scope}: ${failure.message}`).join(" ")}`, + failures, + }) + +export class GcpPromQlService extends Context.Service()( + "@maple/api/services/GcpPromQlService", + { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const httpClient = yield* HttpClient.HttpClient + const edgeCache = yield* EdgeCacheService + const dbExecute = makeDbExecute(database, "GcpPromQlService", toPersistenceError) + + const signingKey = Option.match(env.MAPLE_GCP_SERVICE_ACCOUNT_KEY, { + onNone: () => + Effect.fail( + new GcpPromQlUnavailableError({ + message: + "Live Google Cloud queries are not available on this Maple deployment: it has no Google service account key (MAPLE_GCP_SERVICE_ACCOUNT_KEY).", + }), + ), + onSome: Effect.succeed, + }) + + // Tokens are kept for the isolate's lifetime. Only settled values are kept: a pending + // sign-in shared across requests would resume in another request's I/O context. + let mapleToken: { readonly token: Redacted.Redacted; readonly until: number } | undefined + const readers = new Map() + + const readerFor = Effect.fnUntraced(function* ( + key: Redacted.Redacted, + connector: Connector, + ) { + const now = yield* Clock.currentTimeMillis + const cached = readers.get(connector.id) + if (cached !== undefined && cached.until > now) return cached.reader + if (mapleToken === undefined || mapleToken.until <= now) { + const signer = yield* importServiceAccountKey(key) + mapleToken = { + token: yield* fetchMapleAccessToken(httpClient, signer, now), + until: now + TOKEN_REUSE_MS, + } + } + const reader = yield* impersonateReader(httpClient, mapleToken.token, connector) + readers.set(connector.id, { reader, until: now + TOKEN_REUSE_MS }) + return reader + }) + + const searchProjects = Effect.fnUntraced(function* ( + key: Redacted.Redacted, + connector: Connector, + ) { + const reader = yield* readerFor(key, connector) + // The id is validated when the connector is created; encoding keeps it inside the path. + const scope = `${connector.scopeType}s/${encodeURIComponent(connector.scopeId)}` + const projects: Array = [] + let pageToken: string | undefined + for (let page = 0; page < MAX_PROJECT_PAGES; page++) { + const body = yield* searchResourcesPage(httpClient, reader, scope, pageToken, [ + GCP_PROJECT_ASSET_TYPE, + ]) + for (const result of body.results ?? []) { + const projectId = mapGcpResource(result)?.projectId + // A project pending deletion answers nothing but errors. + if ((result.state ?? "ACTIVE") === "ACTIVE" && isProjectId(projectId)) + projects.push(projectId) + } + pageToken = body.nextPageToken + if (pageToken === undefined) break + } + return projects + }) + + /** + * Every project the org's connections reach, each with the connection that reads it, and + * the connections whose projects could not be listed. + */ + const targetsOf = Effect.fnUntraced(function* (orgId: OrgId, key: Redacted.Redacted) { + const connectors = yield* dbExecute((db) => + db.run( + PG.from(GcpConnectors) + .select("id", "scopeType", "scopeId", "projectId", "lastMetricsReceivedAt") + .where(($) => [$.orgId.eq(orgId)]) + .orderBy(($) => [ + [$.createdAt, "asc"], + [$.id, "asc"], + ]), + ), + ) + if (connectors.length === 0) { + return yield* new IntegrationsNotConnectedError({ + message: + "No Google Cloud connection yet. Connect a project, folder or organization first.", + }) + } + const listed = yield* Effect.forEach( + connectors, + (connector) => + connector.scopeType === "project" + ? Effect.succeed({ + connector, + projects: isProjectId(connector.scopeId) ? [connector.scopeId] : [], + }) + : edgeCache + .getOrCompute( + { + bucket: "gcp-projects", + key: `${orgId}:${connector.id}`, + ttlSeconds: HOUR_SECONDS, + schema: Schema.Array(GcpProjectId), + }, + searchProjects(key, connector), + ) + .pipe( + Effect.map(({ value }) => ({ connector, projects: value })), + Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => + Effect.succeed({ + connector, + projects: [], + failure: { + scope: scopeName(connector), + message: describeGcpApiError(error, connector, true), + }, + }), + ), + ), + { concurrency: CONCURRENCY }, + ) + // Overlapping connections read a project through the oldest. + const projects = new Map() + for (const { connector, projects: ids } of listed) { + for (const id of ids) if (!projects.has(id)) projects.set(id, connector) + } + return { + projects, + failures: listed.flatMap((entry) => ("failure" in entry ? [entry.failure] : [])), + } + }) + + const failure = ( + projectId: GcpProjectId, + connector: Connector, + error: GcpApiError, + ): GcpPromQlFailure => ({ + scope: `projects/${projectId}`, + message: describeGcpApiError(error, connector, true), + }) + + const runQuery = Effect.fnUntraced(function* ( + orgId: OrgId, + requested: ReadonlyArray | undefined, + params: GcpPromQlParams, + endMs: number, + ) { + const key = yield* signingKey + const targets = yield* targetsOf(orgId, key) + let selected = [...targets.projects.entries()] + if (requested !== undefined) { + const unknown = requested.filter((projectId) => !targets.projects.has(projectId)) + if (unknown.length > 0) return yield* notReadable(unknown) + selected = selected.filter(([projectId]) => requested.includes(projectId)) + } else if (selected.length > GCP_PROMQL_MAX_PROJECTS) { + return yield* new GcpPromQlTooManyProjectsError({ + message: `The Google Cloud connections reach ${selected.length} projects, and one query reads at most ${GCP_PROMQL_MAX_PROJECTS}. Choose the projects to query.`, + projects: selected.map(([projectId]) => projectId), + }) + } + const resultType = "time" in params ? "vector" : "matrix" + if (selected.length === 0) { + if (targets.failures.length > 0) return yield* nothingRead(targets.failures) + return new GcpPromQlResult({ resultType, series: [], projects: [], failures: [] }) + } + selected.sort(([a], [b]) => a.localeCompare(b)) + yield* Effect.annotateCurrentSpan({ "maple.gcp.projects": selected.length }) + + const compute = Effect.gen(function* () { + const answers = yield* Effect.forEach( + selected, + ([projectId, connector]) => + readerFor(key, connector).pipe( + Effect.flatMap((reader) => + promQlQuery(httpClient, reader, projectId, params), + ), + Effect.map((series) => ({ projectId, series })), + Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => + Effect.succeed({ + projectId, + failure: failure(projectId, connector, error), + }), + ), + ), + { concurrency: CONCURRENCY }, + ) + const failures = answers.flatMap((answer) => + "failure" in answer ? [answer.failure] : [], + ) + const answered = answers.flatMap((answer) => ("series" in answer ? [answer] : [])) + // Not cached: a fixed setup is read again on the next try. + if (answered.length === 0) return yield* nothingRead([...targets.failures, ...failures]) + return new GcpPromQlResult({ + resultType, + // Identical label sets from two projects stay apart. + series: answered.flatMap(({ projectId, series }) => + series.map((one) => ({ + labels: { ...one.labels, project_id: one.labels.project_id ?? projectId }, + points: one.points, + })), + ), + projects: answered.map(({ projectId }) => projectId), + failures, + }) + }) + const now = yield* Clock.currentTimeMillis + const { value } = yield* edgeCache.getOrCompute( + { + bucket: "gcp-promql", + key: JSON.stringify([orgId, selected.map(([projectId]) => projectId), params]), + ttlSeconds: (result) => + result.failures.length > 0 || endMs > now - FRESH_MS ? 60 : 600, + schema: GcpPromQlResult, + }, + compute, + ) + return targets.failures.length === 0 + ? value + : new GcpPromQlResult({ ...value, failures: [...targets.failures, ...value.failures] }) + }) + + const queryRange = Effect.fn("GcpPromQlService.queryRange")(function* ( + orgId: OrgId, + request: GcpPromQlRangeRequest, + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + const step = request.stepSeconds + // Aligned to the step, so refreshes inside it share a cache entry. + const start = Math.floor(gcpPromQlTimeMs(request.startTime) / 1000 / step) * step + const end = Math.ceil(gcpPromQlTimeMs(request.endTime) / 1000 / step) * step + return yield* runQuery( + orgId, + request.projects, + { query: request.query, start, end, step }, + end * 1000, + ) + }) + + const queryInstant = Effect.fn("GcpPromQlService.queryInstant")(function* ( + orgId: OrgId, + request: GcpPromQlInstantRequest, + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + const time = Math.floor(gcpPromQlTimeMs(request.time) / 60_000) * 60 + return yield* runQuery(orgId, request.projects, { query: request.query, time }, time * 1000) + }) + + /** One project to read discovery data from: the one named, or the first. */ + const discoveryProject = Effect.fnUntraced(function* ( + orgId: OrgId, + project: GcpProjectId | undefined, + ) { + const key = yield* signingKey + const targets = yield* targetsOf(orgId, key) + const projectId = project ?? targets.projects.keys().next().value + const connector = projectId === undefined ? undefined : targets.projects.get(projectId) + if (project !== undefined && connector === undefined) return yield* notReadable([project]) + if (projectId === undefined || connector === undefined) + return yield* nothingRead(targets.failures) + return { + projectId, + reader: readerFor(key, connector), + failed: (error: GcpApiError) => nothingRead([failure(projectId, connector, error)]), + } + }) + + const searchMetrics = Effect.fn("GcpPromQlService.searchMetrics")(function* ( + orgId: OrgId, + request: Parameters[1], + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + const target = yield* discoveryProject(orgId, request.project) + const { value: names } = yield* edgeCache + .getOrCompute( + { + bucket: "gcp-metric-names", + key: `${orgId}:${target.projectId}`, + ttlSeconds: HOUR_SECONDS, + schema: Schema.Array(Schema.String), + }, + target.reader.pipe( + Effect.flatMap((reader) => + listPromQlMetricNames(httpClient, reader, target.projectId), + ), + ), + ) + .pipe(Effect.mapError(target.failed)) + const search = request.search.toLowerCase() + return new GcpMetricSearchResponse({ + projectId: target.projectId, + metrics: names + .filter((name) => name.toLowerCase().includes(search)) + .sort() + .slice(0, request.limit ?? 50), + }) + }) + + const describeMetric = Effect.fn("GcpPromQlService.describeMetric")(function* ( + orgId: OrgId, + request: Parameters[1], + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + const target = yield* discoveryProject(orgId, request.project) + const find = (type: string) => + target.reader.pipe( + Effect.flatMap((reader) => + getMetricDescriptor(httpClient, reader, target.projectId, type), + ), + Effect.mapError(target.failed), + ) + const descriptor = + (yield* find(request.type)) ?? + (DISTRIBUTION_SUFFIX.test(request.type) + ? yield* find(request.type.replace(DISTRIBUTION_SUFFIX, "")) + : undefined) + if (descriptor === undefined) { + return yield* new IntegrationsNotFoundError({ + message: `Project ${target.projectId} has no metric type ${request.type}.`, + }) + } + return new GcpMetricDescriptor({ + type: descriptor.type, + metricKind: descriptor.metricKind ?? "", + valueType: descriptor.valueType ?? "", + unit: descriptor.unit ?? "", + description: descriptor.description ?? "", + monitoredResourceTypes: descriptor.monitoredResourceTypes ?? [], + labels: (descriptor.labels ?? []).map((label) => ({ + key: label.key, + description: label.description ?? "", + })), + }) + }) + + return { queryRange, queryInstant, searchMetrics, describeMetric } satisfies GcpPromQlServiceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(FetchHttpClient.layer)) +} diff --git a/packages/backend/src/services/integrations/gcp/api.ts b/packages/backend/src/services/integrations/gcp/api.ts index 98e53c2948..efedf8525d 100644 --- a/packages/backend/src/services/integrations/gcp/api.ts +++ b/packages/backend/src/services/integrations/gcp/api.ts @@ -4,7 +4,8 @@ * here may leak a credential: tokens stay `Redacted`, and a failure carries the HTTP status and * Google's error code and reason only, never a response body, transport error or schema issue. * The one sentence of Google's that is read, to tell a missing billing account from a missing - * role, chooses a reason and is dropped. + * role, chooses a reason and is dropped. The exception is a refused PromQL query: Google's + * explanation of the caller's own query is passed on. * Untraced on purpose: the HTTP client records a span per request, and "not set up yet" is not an * exception. */ @@ -15,6 +16,7 @@ import { type GcpMetric, type GcpMetricGroup, } from "@maple/domain/gcp-metrics" +import { GcpPromQlInvalidQueryError } from "@maple/domain/http" import type { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" import { Duration, Effect, Option, Redacted, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" @@ -116,19 +118,25 @@ const errorReason = (body: unknown, kind: GcpApiError["kind"]): string | undefin .flatMap((detail) => Option.toArray(decodeErrorInfo(detail)))[0]?.reason ?? (kind === "denied" ? reasonFromMessage(body) : undefined) -/** One JSON call. `api` names the Google service in the error. */ -const call = ( +/** + * One JSON call. `api` names the Google service in the error. `refuse` turns a failed answer + * into the caller's own error instead. + */ +const call = ( httpClient: HttpClient.HttpClient, api: GcpApiError["api"], request: HttpClientRequest.HttpClientRequest, decode: (body: unknown) => Effect.Effect, -): Effect.Effect => { + refuse?: (status: number, body: unknown) => E | undefined, +): Effect.Effect => { const failed = (what: string) => () => Effect.fail(new GcpApiError({ message: `${api} ${what}`, api, kind: "upstream" })) return Effect.gen(function* () { const response = yield* httpClient.execute(request) const body = yield* response.json.pipe(Effect.orElseSucceed(() => null)) if (response.status >= 300) { + const refused = refuse?.(response.status, body) + if (refused !== undefined) return yield* Effect.fail(refused) const kind = errorKind(response.status) const reason = errorReason(body, kind) return yield* new GcpApiError({ @@ -383,6 +391,140 @@ export const listTimeSeriesPage = ( decodeTimeSeriesPage, ) +// Cloud Monitoring PromQL + +/** Google's own limit is 120 s; this one answers before `REQUEST_TIMEOUT` gives up. */ +const PROMQL_TIMEOUT = "18s" + +const promQlUrl = (projectId: GcpProjectId, path: string) => + `https://monitoring.googleapis.com/v1/projects/${projectId}/location/global/prometheus/api/v1/${path}` + +const Labels = Schema.Record(Schema.String, Schema.String) +/** Prometheus sends values as strings: `"1.5"`, `"NaN"`, `"+Inf"`. */ +const Sample = Schema.Tuple([Schema.Number, Schema.String]) + +const decodePromQlResult = Schema.decodeUnknownEffect( + Schema.Struct({ + data: Schema.Union([ + Schema.Struct({ + resultType: Schema.Literal("matrix"), + result: Schema.Array(Schema.Struct({ metric: Labels, values: Schema.Array(Sample) })), + }), + Schema.Struct({ + resultType: Schema.Literal("vector"), + result: Schema.Array(Schema.Struct({ metric: Labels, value: Sample })), + }), + ]), + }), +) + +const toPoints = (samples: ReadonlyArray): Array => + samples.flatMap(([time, raw]) => { + const value = Number(raw) + return Number.isFinite(value) ? [[time, value] as const] : [] + }) + +const decodePromQlError = Schema.decodeUnknownOption( + Schema.Struct({ status: Schema.Literal("error"), errorType: Schema.String, error: Schema.String }), +) + +/** Google's answer to a query it cannot run: `bad_data`, `invalid_argument` or `execution`. */ +const refuseQuery = (projectId: GcpProjectId) => (status: number, body: unknown) => { + if (status !== 400 && status !== 422) return undefined + return Option.match(decodePromQlError(body), { + onNone: () => undefined, + onSome: ({ error }) => new GcpPromQlInvalidQueryError({ message: error.slice(0, 1_000), projectId }), + }) +} + +export type GcpPromQlParams = + | { readonly query: string; readonly time: number } + | { readonly query: string; readonly start: number; readonly end: number; readonly step: number } + +/** One instant or range query against one project, times in epoch seconds. */ +export const promQlQuery = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, + params: GcpPromQlParams, +) => + call( + httpClient, + "Cloud Monitoring", + HttpClientRequest.get(promQlUrl(projectId, "time" in params ? "query" : "query_range")).pipe( + asReader(reader), + HttpClientRequest.setUrlParams({ ...params, timeout: PROMQL_TIMEOUT }), + ), + decodePromQlResult, + refuseQuery(projectId), + ).pipe( + Effect.map(({ data }) => + data.resultType === "matrix" + ? data.result.map((series) => ({ labels: series.metric, points: toPoints(series.values) })) + : data.result.map((series) => ({ labels: series.metric, points: toPoints([series.value]) })), + ), + ) + +const decodeNames = Schema.decodeUnknownEffect(Schema.Struct({ data: Schema.Array(Schema.String) })) + +/** + * Every metric type the project has a descriptor for, data or not. Each comes twice, in PromQL's + * UTF-8 form and the legacy `run_googleapis_com:request_count` form; only the first is kept. + */ +export const listPromQlMetricNames = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, +) => + call( + httpClient, + "Cloud Monitoring", + HttpClientRequest.get(promQlUrl(projectId, "label/__name__/values")).pipe(asReader(reader)), + decodeNames, + ).pipe(Effect.map(({ data }) => data.filter((name) => !name.includes(":")))) + +const decodeMetricDescriptors = Schema.decodeUnknownEffect( + Schema.Struct({ + metricDescriptors: Schema.optionalKey( + Schema.Array( + Schema.Struct({ + type: Schema.String, + metricKind: Schema.optionalKey(Schema.String), + valueType: Schema.optionalKey(Schema.String), + unit: Schema.optionalKey(Schema.String), + description: Schema.optionalKey(Schema.String), + monitoredResourceTypes: Schema.optionalKey(Schema.Array(Schema.String)), + labels: Schema.optionalKey( + Schema.Array( + Schema.Struct({ + key: Schema.String, + description: Schema.optionalKey(Schema.String), + }), + ), + ), + }), + ), + ), + }), +) + +/** The descriptor of one metric type, or undefined when the project has none. */ +export const getMetricDescriptor = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, + /** Validated against `GcpMetricType`, which leaves no quote to break out of the filter. */ + metricType: string, +) => + call( + httpClient, + "Cloud Monitoring", + HttpClientRequest.get( + `https://monitoring.googleapis.com/v3/projects/${projectId}/metricDescriptors`, + ).pipe(asReader(reader), HttpClientRequest.setUrlParams({ filter: `metric.type = "${metricType}"` })), + decodeMetricDescriptors, + ).pipe(Effect.map((body) => body.metricDescriptors?.find((descriptor) => descriptor.type === metricType))) + // Cloud Asset Inventory const ResourceSearchResult = Schema.Struct({ @@ -412,6 +554,7 @@ export const searchResourcesPage = ( reader: GcpReader, scope: string, pageToken: string | undefined, + assetTypes: ReadonlyArray = GCP_ASSET_TYPES, ) => call( httpClient, @@ -419,7 +562,7 @@ export const searchResourcesPage = ( HttpClientRequest.get(`https://cloudasset.googleapis.com/v1/${scope}:searchAllResources`).pipe( asReader(reader), HttpClientRequest.setUrlParams({ - assetTypes: GCP_ASSET_TYPES, + assetTypes, readMask: Object.keys(ResourceSearchResult.fields).join(","), pageSize: RESOURCE_PAGE_SIZE, pageToken, diff --git a/packages/backend/src/services/integrations/gcp/explain.ts b/packages/backend/src/services/integrations/gcp/explain.ts new file mode 100644 index 0000000000..e75c23c2c4 --- /dev/null +++ b/packages/backend/src/services/integrations/gcp/explain.ts @@ -0,0 +1,56 @@ +import type { GcpConnectorRow } from "@maple/db/tables" +import type { GcpApiError } from "./api" + +const RETRIES = "Maple retries in 5 minutes." + +type ExplainedConnector = Pick< + GcpConnectorRow, + "scopeType" | "scopeId" | "projectId" | "lastMetricsReceivedAt" +> + +/** + * Why a failed call failed and what to do about it, in the customer's terms. `live` is a query a + * person is waiting on, which nothing retries; otherwise it is the poller's. + */ +export const explainGcpApiError = (error: GcpApiError, connector: ExplainedConnector, live = false) => { + // The host project is the one Google checks for billing, enabled APIs and quota. A project + // scope is its own host, and reads as "the project". + const host = `${connector.scopeType === "project" ? "project" : "host project"} ${connector.projectId}` + const scope = `${connector.scopeType} ${connector.scopeId}` + const retries = live ? "Try again in a few minutes." : RETRIES + if (error.reason === "BILLING_DISABLED") { + return `The ${host} has no active billing account, and ${error.api} only answers for projects that have one. Link one: https://console.cloud.google.com/billing/linkedaccount?project=${connector.projectId}${live ? "" : " Maple retries every 5 minutes."}` + } + if (error.reason === "SERVICE_DISABLED") { + return `The ${error.api} API is switched off in the ${host}. Run the setup script again: it switches the API on.` + } + if (error.kind === "denied" || error.kind === "not_found") { + if (error.api === "IAM Credentials") { + if (live) { + return "Maple can't sign in as this connection's read-only service account, which live queries read with. Run the setup script again: it creates the account and the grant to Maple. A new grant can take a few minutes." + } + // A connection that has been read before lost its account or its grant. + if (connector.lastMetricsReceivedAt !== null) { + return "Maple can no longer sign in as this connection's read-only service account. The account was deleted, or the grant to Maple was removed or is blocked by an organization policy. Run the setup script again: it restores both, or says what blocks it." + } + // Shown both before the script ever ran and for the minute or two after a run in + // which Google does not accept the new grant yet, so it has to read right in both. + return "Maple can't sign in as this connection's read-only service account yet. After a setup run Google needs a few minutes to accept the new grant, and Maple retries every 5 minutes. If this stays, the account does not exist, or the grant to Maple is missing or blocked by an organization policy: run the setup script and read its last lines." + } + return error.api === "Cloud Asset Inventory" + ? `Google denied the resource listing for ${scope}. Run the setup script again: it grants Cloud Asset Viewer.` + : `Google denied Maple's read of ${scope}: the read-only roles are missing. Run the setup script again: it grants them. A new grant can take a few minutes.` + } + // Signing in is counted against Maple's own project, not the host project. + if (error.kind === "rate_limited" && error.api !== "IAM Credentials") { + return `Google rate-limited the ${error.api} API for the ${host}. ${retries} If this repeats, raise that API's quota on the project.` + } + return `Maple's request to Google's ${error.api} API failed. ${retries} If this repeats, write to support@maple.dev.` +} + +/** + * What the customer sees for a failed call: one sentence of cause, one of what to do, and + * Google's status at the end. + */ +export const describeGcpApiError = (error: GcpApiError, connector: ExplainedConnector, live = false) => + `${explainGcpApiError(error, connector, live)} (${error.message})` diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index e2e88a762c..c3233f8547 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -48,6 +48,10 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/ErrorIssueTransitionError", "@maple/http/errors/ErrorValidationError", "@maple/http/errors/GcpMetricsUnavailableError", + "@maple/http/errors/GcpPromQlFailedError", + "@maple/http/errors/GcpPromQlInvalidQueryError", + "@maple/http/errors/GcpPromQlTooManyProjectsError", + "@maple/http/errors/GcpPromQlUnavailableError", "@maple/http/errors/GcpScopeAlreadyConnectedError", "@maple/http/errors/IncidentTriageUnauthorizedError", "@maple/http/errors/IngestAttributeMappingForbiddenError", diff --git a/packages/domain/src/http/gcp-promql.ts b/packages/domain/src/http/gcp-promql.ts new file mode 100644 index 0000000000..0a3785ba52 --- /dev/null +++ b/packages/domain/src/http/gcp-promql.ts @@ -0,0 +1,173 @@ +/** + * Live PromQL over a customer's Google Cloud Monitoring metrics, read through Google's + * Prometheus-compatible API with each connection's read-only account. Nothing is stored. One call + * per project: an aggregation never spans projects, and every series carries `project_id`. + */ +import { Schema } from "effect" +import { GcpProjectId } from "../primitives" +import { TinybirdDateTime } from "../query-engine" +import { HttpTaggedError } from "./error-policy" +import { BucketSeconds } from "./query-engine" + +/** Projects one query reads. Above it the caller names the projects. */ +export const GCP_PROMQL_MAX_PROJECTS = 20 +export const GCP_PROMQL_MAX_QUERY_LENGTH = 4_000 +export const GCP_PROMQL_MIN_STEP_SECONDS = 60 +export const GCP_METRIC_SEARCH_MAX_LIMIT = 200 + +/** `TinybirdDateTime` is UTC without a zone marker. */ +export const gcpPromQlTimeMs = (value: string): number => Date.parse(`${value.replace(" ", "T")}Z`) + +const PromQlQuery = Schema.String.check( + Schema.isTrimmed(), + Schema.isMinLength(1), + Schema.isMaxLength(GCP_PROMQL_MAX_QUERY_LENGTH), +) + +/** A subset of the organization's projects; any other project is refused. */ +const PromQlProjects = Schema.Array(GcpProjectId).check( + Schema.isMinLength(1), + Schema.isMaxLength(GCP_PROMQL_MAX_PROJECTS), +) + +export class GcpPromQlRangeRequest extends Schema.Class("GcpPromQlRangeRequest")( + Schema.Struct({ + query: PromQlQuery, + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + /** Point spacing. Start and end are aligned to it. */ + stepSeconds: BucketSeconds.check(Schema.isGreaterThanOrEqualTo(GCP_PROMQL_MIN_STEP_SECONDS)), + projects: Schema.optionalKey(PromQlProjects), + }).check( + Schema.makeFilter( + (request: { readonly startTime: string; readonly endTime: string }) => { + const extentMs = gcpPromQlTimeMs(request.endTime) - gcpPromQlTimeMs(request.startTime) + // The pattern admits `2026-13-45 99:99:99`, which parses to NaN. + if (Number.isNaN(extentMs)) return "startTime and endTime must be valid datetimes" + return extentMs > 0 || "endTime must be after startTime" + }, + { identifier: "GcpPromQlWindow" }, + ), + ), +) {} + +export class GcpPromQlInstantRequest extends Schema.Class("GcpPromQlInstantRequest")( + { + query: PromQlQuery, + time: TinybirdDateTime.check( + Schema.makeFilter( + (time: string) => !Number.isNaN(gcpPromQlTimeMs(time)) || "time must be a valid datetime", + ), + ), + projects: Schema.optionalKey(PromQlProjects), + }, +) {} + +export const GcpPromQlSeries = Schema.Struct({ + /** Google's labels, `project_id` always among them. */ + labels: Schema.Record(Schema.String, Schema.String), + /** `[epoch seconds, value]`, NaN and infinities dropped. An instant query has one point. */ + points: Schema.Array(Schema.Tuple([Schema.Number, Schema.Number])), +}) +export type GcpPromQlSeries = Schema.Schema.Type + +/** A project or a connection's scope that could not be read, and what to do about it. */ +export const GcpPromQlFailure = Schema.Struct({ + /** `projects/`, `folders/` or `organizations/`. */ + scope: Schema.String, + message: Schema.String, +}) +export type GcpPromQlFailure = Schema.Schema.Type + +export class GcpPromQlResult extends Schema.Class("GcpPromQlResult")({ + resultType: Schema.Literals(["matrix", "vector"]), + series: Schema.Array(GcpPromQlSeries), + /** The projects that answered. */ + projects: Schema.Array(GcpProjectId), + /** What could not be read. The rest is still returned. */ + failures: Schema.Array(GcpPromQlFailure), +}) {} + +/** A Cloud Monitoring metric type, e.g. `run.googleapis.com/request_count`. */ +export const GcpMetricType = Schema.String.check(Schema.isPattern(/^[A-Za-z0-9][\w./-]{0,254}$/u)) + +export class GcpMetricSearchResponse extends Schema.Class("GcpMetricSearchResponse")( + { + /** The project whose metric list was searched. */ + projectId: GcpProjectId, + /** Metric names in PromQL's UTF-8 form, in name order. Distributions end in `_bucket`, `_count`, `_sum`. */ + metrics: Schema.Array(Schema.String), + }, +) {} + +/** What a starter query needs to know about a metric type. */ +export class GcpMetricDescriptor extends Schema.Class("GcpMetricDescriptor")({ + type: Schema.String, + /** `GAUGE`, `DELTA` or `CUMULATIVE`. */ + metricKind: Schema.String, + /** `BOOL`, `INT64`, `DOUBLE`, `STRING`, `DISTRIBUTION` or `MONEY`. */ + valueType: Schema.String, + unit: Schema.String, + description: Schema.String, + /** More than one: a query needs a `monitored_resource` matcher. */ + monitoredResourceTypes: Schema.Array(Schema.String), + labels: Schema.Array(Schema.Struct({ key: Schema.String, description: Schema.String })), +}) {} + +/** Google refused the PromQL itself. The message is Google's. */ +export class GcpPromQlInvalidQueryError extends HttpTaggedError()( + "@maple/http/errors/GcpPromQlInvalidQueryError", + { message: Schema.String, projectId: GcpProjectId }, + { + status: 400, + code: "gcp_promql_invalid", + title: "Invalid PromQL query", + param: "query", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + +/** The connections cover more projects than one query reads. */ +export class GcpPromQlTooManyProjectsError extends HttpTaggedError()( + "@maple/http/errors/GcpPromQlTooManyProjectsError", + { message: Schema.String, projects: Schema.Array(GcpProjectId) }, + { + status: 400, + code: "gcp_promql_too_many_projects", + title: "Too many projects", + param: "projects", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + +/** Nothing could be read: every project or connection failed. */ +export class GcpPromQlFailedError extends HttpTaggedError()( + "@maple/http/errors/GcpPromQlFailedError", + { message: Schema.String, failures: Schema.Array(GcpPromQlFailure) }, + { + status: 409, + code: "gcp_promql_failed", + title: "Google Cloud could not be read", + retry: "never", + recovery: "none", + exposure: "public_message", + }, +) {} + +/** This deployment has no Google identity to read with. */ +export class GcpPromQlUnavailableError extends HttpTaggedError()( + "@maple/http/errors/GcpPromQlUnavailableError", + { message: Schema.String }, + { + status: 409, + code: "gcp_promql_unavailable", + title: "Live Google Cloud queries are not available", + retry: "never", + recovery: "none", + exposure: "public_message", + }, +) {} diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 5c95200fb1..10e77c575e 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -39,6 +39,7 @@ export * from "./pull-request-ref" export * from "./error-policy" export * from "./ingest-attribute-mappings" export * from "./ingest-keys" +export * from "./gcp-promql" export * from "./integrations" export * from "./onboarding" export * from "./org-clickhouse-settings" diff --git a/packages/domain/src/http/integrations.ts b/packages/domain/src/http/integrations.ts index 0c21b90871..a4031d2193 100644 --- a/packages/domain/src/http/integrations.ts +++ b/packages/domain/src/http/integrations.ts @@ -10,6 +10,19 @@ import { } from "../primitives" import { SessionAuthorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" +import { + GCP_METRIC_SEARCH_MAX_LIMIT, + GcpMetricDescriptor, + GcpMetricSearchResponse, + GcpMetricType, + GcpPromQlFailedError, + GcpPromQlInstantRequest, + GcpPromQlInvalidQueryError, + GcpPromQlRangeRequest, + GcpPromQlResult, + GcpPromQlTooManyProjectsError, + GcpPromQlUnavailableError, +} from "./gcp-promql" import { PrReviewOrgSettings, PrReviewRepositoryConfig } from "./pr-review" import { GitCommitSha, @@ -1100,6 +1113,17 @@ export class GcpMetricsUnavailableError extends HttpTaggedError Date: Thu, 15 Oct 2026 01:12:08 +0200 Subject: [PATCH 2/8] test(gcp): live PromQL fan-out, failures, tokens and discovery --- .../integrations/GcpPromQlService.test.ts | 615 ++++++++++++++++++ .../services/integrations/GcpPromQlService.ts | 92 +-- .../src/services/integrations/gcp/api.ts | 64 +- packages/domain/src/http/gcp-promql.test.ts | 48 ++ 4 files changed, 757 insertions(+), 62 deletions(-) create mode 100644 packages/backend/src/services/integrations/GcpPromQlService.test.ts create mode 100644 packages/domain/src/http/gcp-promql.test.ts diff --git a/packages/backend/src/services/integrations/GcpPromQlService.test.ts b/packages/backend/src/services/integrations/GcpPromQlService.test.ts new file mode 100644 index 0000000000..14c5a3fe52 --- /dev/null +++ b/packages/backend/src/services/integrations/GcpPromQlService.test.ts @@ -0,0 +1,615 @@ +import { afterEach, assert, beforeAll, describe, it } from "@effect/vitest" +import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" +import { GcpPromQlInstantRequest, GcpPromQlRangeRequest } from "@maple/domain/http" +import { GcpProjectId, OrgId } from "@maple/domain/primitives" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { FetchHttpClient } from "effect/http" +import { Env } from "@maple/backend/platform/Env" +import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" +import { GcpPromQlService } from "./GcpPromQlService" + +const now = Date.UTC(2026, 9, 15, 12, 0, 0) +const ORG = Schema.decodeSync(OrgId)("org_gcp") +const OTHER_ORG = "org_other" +const project = Schema.decodeSync(GcpProjectId) + +// Connector ids and the reader account each one's setup script creates. +const PROJECT_CONNECTOR = "018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8" +const FOLDER_CONNECTOR = "028f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8" +const OTHER_CONNECTOR = "038f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8" +const readerOf = (connectorId: string, hostProject: string) => + `maple-${connectorId.replaceAll("-", "").slice(0, 24)}@${hostProject}.iam.gserviceaccount.com` +const PROJECT_READER = readerOf(PROJECT_CONNECTOR, "acme-prod") +const FOLDER_READER = readerOf(FOLDER_CONNECTOR, "acme-host") + +let encodedKey = "" + +beforeAll(async () => { + const pair = await crypto.subtle.generateKey( + { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["sign", "verify"], + ) + const body = Buffer.from(await crypto.subtle.exportKey("pkcs8", pair.privateKey)).toString("base64") + encodedKey = Buffer.from( + JSON.stringify({ + private_key_id: "key-1", + private_key: `-----BEGIN PRIVATE KEY-----\n${body.match(/.{1,64}/g)!.join("\n")}\n-----END PRIVATE KEY-----\n`, + client_email: "collector@maple-prod.iam.gserviceaccount.com", + }), + ).toString("base64") +}) + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + +const denied = () => json({ error: { code: 403, status: "PERMISSION_DENIED", message: "denied" } }, 403) + +interface PromQlCall { + readonly project: string + /** `query`, `query_range` or `label/__name__/values`. */ + readonly path: string + readonly params: Record + /** The reader account whose token made the call. */ + readonly reader: string + readonly quotaProject: string | null +} + +interface Calls { + oauth: number + readonly impersonate: Array + readonly promql: Array + readonly descriptors: Array + readonly assets: Array +} + +interface StubOptions { + readonly impersonate?: (account: string) => Response | undefined + readonly promql?: (call: PromQlCall) => Response | undefined + /** Projects under `folders/123`, as `[projectId, state]`. */ + readonly folderProjects?: ReadonlyArray + readonly names?: ReadonlyArray + readonly descriptors?: Record +} + +const matrix = ( + ...result: ReadonlyArray<{ metric: Record; values: Array<[number, string]> }> +) => json({ status: "success", data: { resultType: "matrix", result } }) + +const stubFetch = (calls: Calls, options: StubOptions = {}) => + (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL( + typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url, + ) + const headers = new Headers(init?.headers) + const reader = (headers.get("authorization") ?? "").replace("Bearer reader:", "") + if (url.host === "oauth2.googleapis.com") { + calls.oauth += 1 + return json({ access_token: "ya29.maple", expires_in: 600 }) + } + if (url.host === "iamcredentials.googleapis.com") { + const account = decodeURIComponent( + url.pathname + .replace("/v1/projects/-/serviceAccounts/", "") + .replace(":generateAccessToken", ""), + ) + calls.impersonate.push(account) + return options.impersonate?.(account) ?? json({ accessToken: `reader:${account}` }) + } + if (url.host === "cloudasset.googleapis.com") { + calls.assets.push(`${url.pathname} ${url.searchParams.get("assetTypes")}`) + return json({ + results: (options.folderProjects ?? []).map(([projectId, state]) => ({ + name: `//cloudresourcemanager.googleapis.com/projects/${projectId.length}0001`, + assetType: "cloudresourcemanager.googleapis.com/Project", + state, + additionalAttributes: { projectId }, + })), + }) + } + const promql = /^\/v1\/projects\/([^/]+)\/location\/global\/prometheus\/api\/v1\/(.+)$/.exec( + url.pathname, + ) + if (url.host === "monitoring.googleapis.com" && promql !== null) { + const call = { + project: promql[1]!, + path: promql[2]!, + params: Object.fromEntries(url.searchParams), + reader, + quotaProject: headers.get("x-goog-user-project"), + } + calls.promql.push(call) + if (call.path === "label/__name__/values") + return json({ status: "success", data: options.names ?? [] }) + return options.promql?.(call) ?? matrix() + } + if (url.host === "monitoring.googleapis.com" && url.pathname.endsWith("/metricDescriptors")) { + const type = /metric\.type = "([^"]+)"/.exec(url.searchParams.get("filter") ?? "")?.[1] ?? "" + calls.descriptors.push(type) + const descriptor = options.descriptors?.[type] + return json(descriptor === undefined ? {} : { metricDescriptors: [descriptor] }) + } + throw new Error(`unexpected request to ${url.href}`) + }) as typeof fetch + +const makeCalls = (): Calls => ({ oauth: 0, impersonate: [], promql: [], descriptors: [], assets: [] }) + +const run = ( + testDb: TestDb, + stub: typeof fetch, + effect: Effect.Effect, + options: { readonly key?: string | null } = {}, +) => { + const key = options.key === undefined ? encodedKey : options.key + return Effect.gen(function* () { + yield* TestClock.setTime(now) + return yield* effect + }).pipe( + Effect.provide( + Layer.effect(GcpPromQlService, GcpPromQlService.make).pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, stub)), + Layer.provide(EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive))), + Layer.provide(testDb.layer), + Layer.provide(Env.layer), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 9).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + ...(key === null ? undefined : { MAPLE_GCP_SERVICE_ACCOUNT_KEY: key }), + }), + ), + ), + ), + ), + ) +} + +const insertConnector = ( + testDb: TestDb, + id: string, + options: { + readonly orgId?: string + readonly projectId?: string + readonly folder?: string + readonly metricsEnabled?: boolean + } = {}, +) => { + const projectId = options.projectId ?? "acme-prod" + return Effect.promise(() => + executeSql( + testDb, + `INSERT INTO gcp_connectors + (id, org_id, scope_type, scope_id, project_id, secret_ciphertext, secret_iv, secret_tag, + secret_hash, metrics_enabled, created_by, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, 'c', 'i', 't', $1, $6, 'user_1', now(), now())`, + [ + id, + options.orgId ?? ORG, + options.folder === undefined ? "project" : "folder", + options.folder ?? projectId, + projectId, + options.metricsEnabled ?? false, + ], + ), + ) +} + +/** The org's two connections: project `acme-prod`, and folder 123 hosted in `acme-host`. */ +const insertConnectors = (testDb: TestDb) => + Effect.all([ + insertConnector(testDb, PROJECT_CONNECTOR), + insertConnector(testDb, FOLDER_CONNECTOR, { + projectId: "acme-host", + folder: "123", + metricsEnabled: true, + }), + insertConnector(testDb, OTHER_CONNECTOR, { orgId: OTHER_ORG, projectId: "rival-prod" }), + ]) + +const FOLDER_PROJECTS = [ + ["acme-shop", "ACTIVE"], + ["acme-data", "ACTIVE"], + ["acme-gone", "DELETE_REQUESTED"], +] as const + +const range = (query: string, projects?: ReadonlyArray) => + new GcpPromQlRangeRequest({ + query, + startTime: "2026-10-15 10:59:30", + endTime: "2026-10-15 11:59:30", + stepSeconds: 600, + ...(projects === undefined ? undefined : { projects: projects.map((id) => project(id)) }), + }) + +const queryRange = (request: GcpPromQlRangeRequest) => + GcpPromQlService.use((gcp) => gcp.queryRange(ORG, request)) + +const T0 = Date.UTC(2026, 9, 15, 11, 0) / 1000 + +describe("GcpPromQlService.queryRange", () => { + it.effect("reads every project of the org's connections, each with its connection's reader", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + promql: (call) => + call.project === "acme-prod" + ? // Google left out `project_id`, and two points are not numbers. + matrix({ + metric: { service: "checkout" }, + values: [ + [T0, "3"], + [T0 + 600, "NaN"], + [T0 + 1200, "+Inf"], + [T0 + 1800, "1.5"], + ], + }) + : matrix({ + metric: { service: "checkout", project_id: call.project }, + values: [[T0, "7"]], + }), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const result = yield* queryRange(range("sum by (service) (rate(x[5m]))")) + + assert.deepStrictEqual>(result.projects, ["acme-data", "acme-prod", "acme-shop"]) + assert.deepStrictEqual(result.failures, []) + assert.strictEqual(result.resultType, "matrix") + // Identical label sets from different projects stay apart by `project_id`. + assert.sameDeepMembers( + result.series.map((series) => ({ labels: series.labels, points: series.points })), + [ + { labels: { service: "checkout", project_id: "acme-data" }, points: [[T0, 7]] }, + { + labels: { service: "checkout", project_id: "acme-prod" }, + points: [ + [T0, 3], + [T0 + 1800, 1.5], + ], + }, + { labels: { service: "checkout", project_id: "acme-shop" }, points: [[T0, 7]] }, + ], + ) + + // Only the folder is searched, for projects only; the other org's connection is never read. + assert.deepStrictEqual(calls.assets, [ + "/v1/folders/123:searchAllResources cloudresourcemanager.googleapis.com/Project", + ]) + assert.sameDeepMembers( + calls.promql.map(({ project, reader, quotaProject }) => ({ + project, + reader, + quotaProject, + })), + [ + { project: "acme-prod", reader: PROJECT_READER, quotaProject: "acme-prod" }, + { project: "acme-shop", reader: FOLDER_READER, quotaProject: "acme-host" }, + { project: "acme-data", reader: FOLDER_READER, quotaProject: "acme-host" }, + ], + ) + // The window is aligned to the step. + assert.deepInclude(calls.promql[0]!.params, { + query: "sum by (service) (rate(x[5m]))", + start: String(T0 - 600), + end: String(T0 + 3600), + step: "600", + }) + assert.strictEqual(calls.promql[0]!.path, "query_range") + }), + ) + }) + + it.effect("reads only the projects asked for, and refuses one the connections do not reach", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { folderProjects: FOLDER_PROJECTS }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const result = yield* queryRange(range("up", ["acme-shop"])) + assert.deepStrictEqual>(result.projects, ["acme-shop"]) + assert.deepStrictEqual( + calls.promql.map((call) => call.project), + ["acme-shop"], + ) + + for (const outside of ["rival-prod", "acme-gone"]) { + const error = yield* Effect.flip(queryRange(range("up", ["acme-shop", outside]))) + assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsValidationError") + assert.include(error.message, outside) + } + assert.lengthOf(calls.promql, 1) + }), + ) + }) + + it.effect("refuses to read more than 20 projects unless they are chosen", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const many = Array.from({ length: 21 }, (_, index) => [`acme-team-${index + 10}`, "ACTIVE"] as const) + return run( + testDb, + stubFetch(calls, { folderProjects: many }), + Effect.gen(function* () { + yield* insertConnector(testDb, FOLDER_CONNECTOR, { projectId: "acme-host", folder: "123" }) + const error = yield* Effect.flip(queryRange(range("up"))) + assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlTooManyProjectsError") + assert.lengthOf( + error._tag === "@maple/http/errors/GcpPromQlTooManyProjectsError" ? error.projects : [], + 21, + ) + assert.lengthOf(calls.promql, 0) + + const chosen = yield* queryRange(range("up", ["acme-team-10", "acme-team-30"])) + assert.deepStrictEqual>(chosen.projects, ["acme-team-10", "acme-team-30"]) + }), + ) + }) + + it.effect("returns what answered beside what failed, and fails only when nothing answered", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + promql: (call) => + call.project === "acme-prod" + ? matrix({ metric: { project_id: "acme-prod" }, values: [[T0, "1"]] }) + : denied(), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const result = yield* queryRange(range("up")) + assert.deepStrictEqual>(result.projects, ["acme-prod"]) + assert.lengthOf(result.series, 1) + assert.deepStrictEqual( + result.failures.map((failure) => failure.scope), + ["projects/acme-data", "projects/acme-shop"], + ) + assert.strictEqual( + result.failures[0]!.message, + "Google denied Maple's read of folder 123: the read-only roles are missing. Run the setup script again: it grants them. A new grant can take a few minutes. (Cloud Monitoring returned 403 PERMISSION_DENIED)", + ) + + const error = yield* Effect.flip(queryRange(range("up", ["acme-data", "acme-shop"]))) + assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlFailedError") + }), + ) + }) + + it.effect("reports a connection without a read-only account on its own, and reads the rest", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + impersonate: (account) => (account === FOLDER_READER ? denied() : undefined), + promql: (call) => matrix({ metric: { project_id: call.project }, values: [[T0, "1"]] }), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const result = yield* queryRange(range("up")) + assert.deepStrictEqual>(result.projects, ["acme-prod"]) + assert.deepStrictEqual(result.failures, [ + { + scope: "folders/123", + message: + "Maple can't sign in as this connection's read-only service account, which live queries read with. Run the setup script again: it creates the account and the grant to Maple. A new grant can take a few minutes. (IAM Credentials returned 403 PERMISSION_DENIED)", + }, + ]) + }), + ) + }) + + it.effect("passes Google's refusal of the query on as an invalid query", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const refusal = + "invalid parameter \"query\": substitute queries failed: must specify a label matcher on the 'monitored_resource' label" + return run( + testDb, + stubFetch(calls, { + promql: () => json({ status: "error", errorType: "bad_data", error: refusal }, 400), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip( + queryRange(range('{"serviceruntime.googleapis.com/api/request_count"}')), + ) + assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlInvalidQueryError") + assert.strictEqual(error.message, refusal) + }), + ) + }) + + it.effect("signs in once and reuses the tokens for later queries", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { folderProjects: FOLDER_PROJECTS }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + yield* queryRange(range("up")) + yield* queryRange(range("down")) + assert.strictEqual(calls.oauth, 1) + assert.sameMembers(calls.impersonate, [PROJECT_READER, FOLDER_READER]) + // The folder's projects are searched once an hour, not per query. + assert.lengthOf(calls.assets, 1) + assert.lengthOf(calls.promql, 6) + + // Ten minutes on, both tokens have expired. + yield* TestClock.adjust("10 minutes") + yield* queryRange(range("sideways")) + assert.strictEqual(calls.oauth, 2) + assert.lengthOf(calls.impersonate, 4) + }), + ) + }) + + it.effect("answers unavailable without a key, and not connected without a connection", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return Effect.gen(function* () { + const unavailable = yield* run( + testDb, + stubFetch(calls), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + return yield* Effect.flip(queryRange(range("up"))) + }), + { key: null }, + ) + assert.strictEqual(unavailable._tag, "@maple/http/errors/GcpPromQlUnavailableError") + + const notConnected = yield* run( + testDb, + stubFetch(calls), + Effect.flip( + GcpPromQlService.use((gcp) => + gcp.queryRange(Schema.decodeSync(OrgId)("org_empty"), range("up")), + ), + ), + ) + assert.strictEqual(notConnected._tag, "@maple/http/errors/IntegrationsNotConnectedError") + assert.deepStrictEqual(calls, makeCalls()) + }) + }) +}) + +describe("GcpPromQlService.queryInstant", () => { + it.effect("reads one point per series at the minute, a scalar as one series", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + promql: () => json({ status: "success", data: { resultType: "scalar", result: [T0, "2"] } }), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const result = yield* GcpPromQlService.use((gcp) => + gcp.queryInstant( + ORG, + new GcpPromQlInstantRequest({ query: "1 + 1", time: "2026-10-15 11:00:42" }), + ), + ) + assert.deepInclude(calls.promql[0]!, { path: "query" }) + assert.deepInclude(calls.promql[0]!.params, { query: "1 + 1", time: String(T0) }) + assert.strictEqual(result.resultType, "vector") + assert.deepStrictEqual(result.series, [ + { labels: { project_id: "acme-prod" }, points: [[T0, 2]] }, + ]) + }), + ) + }) +}) + +describe("GcpPromQlService discovery", () => { + it.effect( + "searches one project's UTF-8 metric names and describes a distribution by its PromQL name", + () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + names: [ + "run.googleapis.com/request_count", + "run_googleapis_com:request_count", + "run.googleapis.com/request_latencies_bucket", + "compute.googleapis.com/instance/cpu/utilization", + ], + descriptors: { + "run.googleapis.com/request_latencies": { + type: "run.googleapis.com/request_latencies", + metricKind: "DELTA", + valueType: "DISTRIBUTION", + unit: "ms", + description: "Distribution of request latency.", + monitoredResourceTypes: ["cloud_run_instance", "cloud_run_revision"], + labels: [ + { key: "response_code", description: "Response code of a request." }, + { key: "route" }, + ], + }, + }, + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const search = yield* GcpPromQlService.use((gcp) => + gcp.searchMetrics(ORG, { search: "RUN.googleapis" }), + ) + assert.deepStrictEqual(search.projectId, "acme-prod") + assert.deepStrictEqual(search.metrics, [ + "run.googleapis.com/request_count", + "run.googleapis.com/request_latencies_bucket", + ]) + const limited = yield* GcpPromQlService.use((gcp) => + gcp.searchMetrics(ORG, { search: "run", limit: 1 }), + ) + assert.deepStrictEqual(limited.metrics, ["run.googleapis.com/request_count"]) + // The name list is read once and searched from the cache after. + assert.lengthOf(calls.promql, 1) + + const descriptor = yield* GcpPromQlService.use((gcp) => + gcp.describeMetric(ORG, { type: "run.googleapis.com/request_latencies_bucket" }), + ) + assert.deepInclude(descriptor, { + type: "run.googleapis.com/request_latencies", + metricKind: "DELTA", + valueType: "DISTRIBUTION", + unit: "ms", + }) + assert.deepStrictEqual(descriptor.monitoredResourceTypes, [ + "cloud_run_instance", + "cloud_run_revision", + ]) + assert.deepStrictEqual(descriptor.labels, [ + { key: "response_code", description: "Response code of a request." }, + { key: "route", description: "" }, + ]) + + const missing = yield* Effect.flip( + GcpPromQlService.use((gcp) => + gcp.describeMetric(ORG, { type: "custom.googleapis.com/nothing" }), + ), + ) + assert.strictEqual(missing._tag, "@maple/http/errors/IntegrationsNotFoundError") + const outside = yield* Effect.flip( + GcpPromQlService.use((gcp) => + gcp.describeMetric(ORG, { + type: "run.googleapis.com/request_count", + project: project("rival-prod"), + }), + ), + ) + assert.strictEqual(outside._tag, "@maple/http/errors/IntegrationsValidationError") + }), + ) + }, + ) +}) diff --git a/packages/backend/src/services/integrations/GcpPromQlService.ts b/packages/backend/src/services/integrations/GcpPromQlService.ts index 8aed07bafd..24e6fade41 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.ts @@ -209,38 +209,36 @@ export class GcpPromQlService extends Context.Service - connector.scopeType === "project" - ? Effect.succeed({ - connector, - projects: isProjectId(connector.scopeId) ? [connector.scopeId] : [], - }) - : edgeCache - .getOrCompute( - { - bucket: "gcp-projects", - key: `${orgId}:${connector.id}`, - ttlSeconds: HOUR_SECONDS, - schema: Schema.Array(GcpProjectId), - }, - searchProjects(key, connector), - ) - .pipe( - Effect.map(({ value }) => ({ connector, projects: value })), - Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => - Effect.succeed({ - connector, - projects: [], - failure: { - scope: scopeName(connector), - message: describeGcpApiError(error, connector, true), - }, - }), - ), + // One connection at a time, so a cold isolate signs in as Maple once. + const listed = yield* Effect.forEach(connectors, (connector) => + connector.scopeType === "project" + ? Effect.succeed({ + connector, + projects: isProjectId(connector.scopeId) ? [connector.scopeId] : [], + }) + : edgeCache + .getOrCompute( + { + bucket: "gcp-projects", + key: `${orgId}:${connector.id}`, + ttlSeconds: HOUR_SECONDS, + schema: Schema.Array(GcpProjectId), + }, + searchProjects(key, connector), + ) + .pipe( + Effect.map(({ value }) => ({ connector, projects: value })), + Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => + Effect.succeed({ + connector, + projects: [], + failure: { + scope: scopeName(connector), + message: describeGcpApiError(error, connector, true), + }, + }), ), - { concurrency: CONCURRENCY }, + ), ) // Overlapping connections read a project through the oldest. const projects = new Map() @@ -290,13 +288,29 @@ export class GcpPromQlService extends Context.Service() + const failures: Array = [] + for (const connector of new Set(selected.map(([, connector]) => connector))) { + const reader = yield* readerFor(key, connector).pipe( + Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => { + failures.push({ + scope: scopeName(connector), + message: describeGcpApiError(error, connector, true), + }) + return Effect.succeed(undefined) + }), + ) + if (reader !== undefined) readers.set(connector, reader) + } const answers = yield* Effect.forEach( - selected, - ([projectId, connector]) => - readerFor(key, connector).pipe( - Effect.flatMap((reader) => - promQlQuery(httpClient, reader, projectId, params), - ), + selected.flatMap(([projectId, connector]) => { + const reader = readers.get(connector) + return reader === undefined ? [] : [{ projectId, connector, reader }] + }), + ({ projectId, connector, reader }) => + promQlQuery(httpClient, reader, projectId, params).pipe( Effect.map((series) => ({ projectId, series })), Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => Effect.succeed({ @@ -307,8 +321,8 @@ export class GcpPromQlService extends Context.Service - "failure" in answer ? [answer.failure] : [], + failures.push( + ...answers.flatMap((answer) => ("failure" in answer ? [answer.failure] : [])), ) const answered = answers.flatMap((answer) => ("series" in answer ? [answer] : [])) // Not cached: a fixed setup is read again on the next try. diff --git a/packages/backend/src/services/integrations/gcp/api.ts b/packages/backend/src/services/integrations/gcp/api.ts index efedf8525d..2327c65a03 100644 --- a/packages/backend/src/services/integrations/gcp/api.ts +++ b/packages/backend/src/services/integrations/gcp/api.ts @@ -129,31 +129,36 @@ const call = ( decode: (body: unknown) => Effect.Effect, refuse?: (status: number, body: unknown) => E | undefined, ): Effect.Effect => { - const failed = (what: string) => () => - Effect.fail(new GcpApiError({ message: `${api} ${what}`, api, kind: "upstream" })) + const failed = (what: string) => new GcpApiError({ message: `${api} ${what}`, api, kind: "upstream" }) return Effect.gen(function* () { const response = yield* httpClient.execute(request) const body = yield* response.json.pipe(Effect.orElseSucceed(() => null)) - if (response.status >= 300) { - const refused = refuse?.(response.status, body) - if (refused !== undefined) return yield* Effect.fail(refused) - const kind = errorKind(response.status) - const reason = errorReason(body, kind) - return yield* new GcpApiError({ - message: `${api} returned ${response.status}${errorCode(body)}${reason === undefined ? "" : `, ${reason}`}`, - api, - kind, - ...(reason === undefined ? undefined : { reason }), - }) - } - return yield* decode(body) + return { status: response.status, body } }).pipe( Effect.timeout(REQUEST_TIMEOUT), Effect.catchTags({ - HttpClientError: failed("request failed"), - TimeoutError: failed("request timed out"), - SchemaError: failed("answered with an unexpected body"), + HttpClientError: () => Effect.fail(failed("request failed")), + TimeoutError: () => Effect.fail(failed("request timed out")), }), + Effect.flatMap(({ status, body }) => + Effect.gen(function* () { + if (status >= 300) { + const refused = refuse?.(status, body) + if (refused !== undefined) return yield* Effect.fail(refused) + const kind = errorKind(status) + const reason = errorReason(body, kind) + return yield* new GcpApiError({ + message: `${api} returned ${status}${errorCode(body)}${reason === undefined ? "" : `, ${reason}`}`, + api, + kind, + ...(reason === undefined ? undefined : { reason }), + }) + } + return yield* decode(body).pipe( + Effect.mapError(() => failed("answered with an unexpected body")), + ) + }), + ), Effect.annotateSpans("peer.service", "google-cloud"), ) } @@ -414,6 +419,8 @@ const decodePromQlResult = Schema.decodeUnknownEffect( resultType: Schema.Literal("vector"), result: Schema.Array(Schema.Struct({ metric: Labels, value: Sample })), }), + // `1 + 1`, `time()`: answered as one series without labels. + Schema.Struct({ resultType: Schema.Literal("scalar"), result: Sample }), ]), }), ) @@ -458,11 +465,22 @@ export const promQlQuery = ( decodePromQlResult, refuseQuery(projectId), ).pipe( - Effect.map(({ data }) => - data.resultType === "matrix" - ? data.result.map((series) => ({ labels: series.metric, points: toPoints(series.values) })) - : data.result.map((series) => ({ labels: series.metric, points: toPoints([series.value]) })), - ), + Effect.map(({ data }) => { + switch (data.resultType) { + case "matrix": + return data.result.map((series) => ({ + labels: series.metric, + points: toPoints(series.values), + })) + case "vector": + return data.result.map((series) => ({ + labels: series.metric, + points: toPoints([series.value]), + })) + case "scalar": + return [{ labels: {}, points: toPoints([data.result]) }] + } + }), ) const decodeNames = Schema.decodeUnknownEffect(Schema.Struct({ data: Schema.Array(Schema.String) })) diff --git a/packages/domain/src/http/gcp-promql.test.ts b/packages/domain/src/http/gcp-promql.test.ts new file mode 100644 index 0000000000..6b7b8b149c --- /dev/null +++ b/packages/domain/src/http/gcp-promql.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest" +import { Schema } from "effect" +import { GcpMetricType, GcpPromQlInstantRequest, GcpPromQlRangeRequest } from "./gcp-promql" + +const decodeRange = Schema.decodeUnknownExit(GcpPromQlRangeRequest) +const valid = { + query: "sum(up)", + startTime: "2026-10-15 11:00:00", + endTime: "2026-10-15 12:00:00", + stepSeconds: 60, +} + +describe("GcpPromQlRangeRequest", () => { + it("accepts a query over a window with a step of a minute or more", () => { + expect(decodeRange({ ...valid, projects: ["acme-prod"] })._tag).toBe("Success") + }) + + it.each([ + ["an empty query", { query: "" }], + ["a query over 4,000 characters", { query: `sum(${"x".repeat(4_000)})` }], + ["a step under a minute", { stepSeconds: 30 }], + ["a fractional step", { stepSeconds: 90.5 }], + ["an inverted window", { startTime: valid.endTime, endTime: valid.startTime }], + ["an impossible date", { endTime: "2026-13-45 99:99:99" }], + ["a project id that is not one", { projects: ["../../organizations/1"] }], + ["more than 20 projects", { projects: Array.from({ length: 21 }, (_, i) => `acme-team-${i + 10}`) }], + ])("refuses %s", (_, change) => { + expect(decodeRange({ ...valid, ...change })._tag).toBe("Failure") + }) +}) + +describe("GcpPromQlInstantRequest", () => { + it("refuses an impossible time", () => { + const decode = Schema.decodeUnknownExit(GcpPromQlInstantRequest) + expect(decode({ query: "up", time: "2026-10-15 12:00:00" })._tag).toBe("Success") + expect(decode({ query: "up", time: "2026-02-30 25:00:00" })._tag).toBe("Failure") + }) +}) + +describe("GcpMetricType", () => { + it("admits metric types and nothing that could leave a quoted filter", () => { + const is = Schema.is(GcpMetricType) + expect(is("run.googleapis.com/request_count")).toBe(true) + expect(is("logging.googleapis.com/user/my-metric")).toBe(true) + expect(is('x" OR metric.type = starts_with("')).toBe(false) + expect(is("x\\")).toBe(false) + }) +}) From 177072c4b9255b2513613eb908db82305b86746f Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 01:21:48 +0200 Subject: [PATCH 3/8] fix(gcp): review fixes for live PromQL queries --- .../integrations/GcpMetricsService.ts | 3 +- .../integrations/GcpPromQlService.test.ts | 147 +++++++++++++++--- .../services/integrations/GcpPromQlService.ts | 71 ++++++--- .../src/services/integrations/gcp/api.ts | 26 +++- .../src/services/integrations/gcp/explain.ts | 4 +- packages/domain/src/http/gcp-promql.test.ts | 4 +- packages/domain/src/http/gcp-promql.ts | 36 +++-- 7 files changed, 225 insertions(+), 66 deletions(-) diff --git a/packages/backend/src/services/integrations/GcpMetricsService.ts b/packages/backend/src/services/integrations/GcpMetricsService.ts index fa1269251c..62acca561b 100644 --- a/packages/backend/src/services/integrations/GcpMetricsService.ts +++ b/packages/backend/src/services/integrations/GcpMetricsService.ts @@ -33,7 +33,7 @@ import { type GcpReader, type GcpTimeSeriesQuery, } from "./gcp/api" -import { describeGcpApiError } from "./gcp/explain" +import { describeGcpApiError, RETRIES } from "./gcp/explain" import { mapGcpResource, mapGcpTimeSeries, type GcpMetricRows } from "./gcp/mapping" const MINUTE_MS = 60_000 @@ -71,7 +71,6 @@ const METRICS = GCP_METRIC_GROUPS.flatMap((group) => group.metrics.map((metric) const SYSTEM_USER_ID = Schema.decodeUnknownSync(UserId)("system-gcp-metrics") -const RETRIES = "Maple retries in 5 minutes." const NOT_STORED = `${RETRIES} Nothing to do.` /** What helps a scope too large for one read. A project has no parts to connect instead. */ diff --git a/packages/backend/src/services/integrations/GcpPromQlService.test.ts b/packages/backend/src/services/integrations/GcpPromQlService.test.ts index 14c5a3fe52..425b9d4384 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.test.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.test.ts @@ -77,6 +77,7 @@ interface StubOptions { readonly promql?: (call: PromQlCall) => Response | undefined /** Projects under `folders/123`, as `[projectId, state]`. */ readonly folderProjects?: ReadonlyArray + readonly assets?: () => Response | undefined readonly names?: ReadonlyArray readonly descriptors?: Record } @@ -107,14 +108,17 @@ const stubFetch = (calls: Calls, options: StubOptions = {}) => } if (url.host === "cloudasset.googleapis.com") { calls.assets.push(`${url.pathname} ${url.searchParams.get("assetTypes")}`) - return json({ - results: (options.folderProjects ?? []).map(([projectId, state]) => ({ - name: `//cloudresourcemanager.googleapis.com/projects/${projectId.length}0001`, - assetType: "cloudresourcemanager.googleapis.com/Project", - state, - additionalAttributes: { projectId }, - })), - }) + return ( + options.assets?.() ?? + json({ + results: (options.folderProjects ?? []).map(([projectId, state]) => ({ + name: `//cloudresourcemanager.googleapis.com/projects/${projectId.length}0001`, + assetType: "cloudresourcemanager.googleapis.com/Project", + state, + additionalAttributes: { projectId }, + })), + }) + ) } const promql = /^\/v1\/projects\/([^/]+)\/location\/global\/prometheus\/api\/v1\/(.+)$/.exec( url.pathname, @@ -123,7 +127,10 @@ const stubFetch = (calls: Calls, options: StubOptions = {}) => const call = { project: promql[1]!, path: promql[2]!, - params: Object.fromEntries(url.searchParams), + params: Object.fromEntries([ + ...url.searchParams, + ...new URLSearchParams(await new Response(init?.body ?? null).text()), + ]), reader, quotaProject: headers.get("x-goog-user-project"), } @@ -263,16 +270,36 @@ describe("GcpPromQlService.queryRange", () => { [T0 + 1800, "1.5"], ], }) - : matrix({ - metric: { service: "checkout", project_id: call.project }, - values: [[T0, "7"]], - }), + : // A project's metrics scope can answer for another project: Google's label stays, + // and acme-data, whose scope holds acme-shop, answers acme-shop's series again. + matrix( + { + metric: { + service: "checkout", + project_id: + call.project === "acme-shop" ? "acme-monitored" : call.project, + }, + values: [[T0, "7"]], + }, + ...(call.project === "acme-data" + ? [ + { + metric: { service: "checkout", project_id: "acme-monitored" }, + values: [[T0, "7"]] as Array<[number, string]>, + }, + ] + : []), + ), }), Effect.gen(function* () { yield* insertConnectors(testDb) const result = yield* queryRange(range("sum by (service) (rate(x[5m]))")) - assert.deepStrictEqual>(result.projects, ["acme-data", "acme-prod", "acme-shop"]) + assert.deepStrictEqual>(result.projects, [ + "acme-data", + "acme-prod", + "acme-shop", + ]) assert.deepStrictEqual(result.failures, []) assert.strictEqual(result.resultType, "matrix") // Identical label sets from different projects stay apart by `project_id`. @@ -287,7 +314,7 @@ describe("GcpPromQlService.queryRange", () => { [T0 + 1800, 1.5], ], }, - { labels: { service: "checkout", project_id: "acme-shop" }, points: [[T0, 7]] }, + { labels: { service: "checkout", project_id: "acme-monitored" }, points: [[T0, 7]] }, ], ) @@ -347,22 +374,27 @@ describe("GcpPromQlService.queryRange", () => { it.effect("refuses to read more than 20 projects unless they are chosen", () => { const testDb = createTestDb(trackedDbs) const calls = makeCalls() - const many = Array.from({ length: 21 }, (_, index) => [`acme-team-${index + 10}`, "ACTIVE"] as const) + const many = Array.from({ length: 20 }, (_, index) => [`acme-team-${index + 10}`, "ACTIVE"] as const) return run( testDb, stubFetch(calls, { folderProjects: many }), Effect.gen(function* () { yield* insertConnector(testDb, FOLDER_CONNECTOR, { projectId: "acme-host", folder: "123" }) - const error = yield* Effect.flip(queryRange(range("up"))) + const twenty = yield* queryRange(range("up")) + assert.lengthOf(twenty.projects, 20) + + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const promqlCalls = calls.promql.length + const error = yield* Effect.flip(queryRange(range("down"))) assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlTooManyProjectsError") assert.lengthOf( error._tag === "@maple/http/errors/GcpPromQlTooManyProjectsError" ? error.projects : [], 21, ) - assert.lengthOf(calls.promql, 0) + assert.lengthOf(calls.promql, promqlCalls) - const chosen = yield* queryRange(range("up", ["acme-team-10", "acme-team-30"])) - assert.deepStrictEqual>(chosen.projects, ["acme-team-10", "acme-team-30"]) + const chosen = yield* queryRange(range("up", ["acme-team-10", "acme-prod"])) + assert.deepStrictEqual>(chosen.projects, ["acme-prod", "acme-team-10"]) }), ) }) @@ -395,6 +427,12 @@ describe("GcpPromQlService.queryRange", () => { const error = yield* Effect.flip(queryRange(range("up", ["acme-data", "acme-shop"]))) assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlFailedError") + assert.deepStrictEqual( + error._tag === "@maple/http/errors/GcpPromQlFailedError" + ? error.failures.map((failure) => failure.scope) + : [], + ["projects/acme-data", "projects/acme-shop"], + ) }), ) }) @@ -432,7 +470,10 @@ describe("GcpPromQlService.queryRange", () => { return run( testDb, stubFetch(calls, { - promql: () => json({ status: "error", errorType: "bad_data", error: refusal }, 400), + promql: (call) => + call.params.query === "slow" + ? json({ status: "error", errorType: "execution", error: "query timed out" }, 422) + : json({ status: "error", errorType: "bad_data", error: refusal }, 400), }), Effect.gen(function* () { yield* insertConnector(testDb, PROJECT_CONNECTOR) @@ -441,6 +482,68 @@ describe("GcpPromQlService.queryRange", () => { ) assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlInvalidQueryError") assert.strictEqual(error.message, refusal) + assert.strictEqual( + error._tag === "@maple/http/errors/GcpPromQlInvalidQueryError" ? error.projectId : "", + "acme-prod", + ) + + // A query that ran out of time is the project's failure, not the query's fault. + const slow = yield* Effect.flip(queryRange(range("slow"))) + assert.strictEqual(slow._tag, "@maple/http/errors/GcpPromQlFailedError") + }), + ) + }) + + it.effect("reports a folder whose projects cannot be listed, also on a cached answer", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + assets: denied, + promql: (call) => matrix({ metric: { project_id: call.project }, values: [[T0, "1"]] }), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + for (const _ of [1, 2]) { + const result = yield* queryRange(range("up")) + assert.deepStrictEqual>(result.projects, ["acme-prod"]) + assert.deepStrictEqual( + result.failures.map((failure) => failure.scope), + ["folders/123"], + ) + assert.include(result.failures[0]!.message, "it grants Cloud Asset Viewer") + } + // The second answer came from the cache. + assert.lengthOf(calls.promql, 1) + }), + ) + }) + + it.effect("never serves one organization's cached answer to another", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const other = Schema.decodeSync(OrgId)(OTHER_ORG) + return run( + testDb, + stubFetch(calls, { names: ["run.googleapis.com/request_count"] }), + Effect.gen(function* () { + // Both organizations connected the same project, each with its own reader. + yield* insertConnector(testDb, PROJECT_CONNECTOR) + yield* insertConnector(testDb, OTHER_CONNECTOR, { orgId: OTHER_ORG }) + yield* queryRange(range("up")) + yield* GcpPromQlService.use((gcp) => gcp.queryRange(other, range("up"))) + yield* GcpPromQlService.use((gcp) => gcp.searchMetrics(ORG, { search: "run" })) + yield* GcpPromQlService.use((gcp) => gcp.searchMetrics(other, { search: "run" })) + assert.deepStrictEqual( + calls.promql.map((call) => [call.path, call.reader]), + [ + ["query_range", PROJECT_READER], + ["query_range", readerOf(OTHER_CONNECTOR, "acme-prod")], + ["label/__name__/values", PROJECT_READER], + ["label/__name__/values", readerOf(OTHER_CONNECTOR, "acme-prod")], + ], + ) }), ) }) @@ -563,7 +666,7 @@ describe("GcpPromQlService discovery", () => { const search = yield* GcpPromQlService.use((gcp) => gcp.searchMetrics(ORG, { search: "RUN.googleapis" }), ) - assert.deepStrictEqual(search.projectId, "acme-prod") + assert.strictEqual(search.projectId, "acme-prod") assert.deepStrictEqual(search.metrics, [ "run.googleapis.com/request_count", "run.googleapis.com/request_latencies_bucket", diff --git a/packages/backend/src/services/integrations/GcpPromQlService.ts b/packages/backend/src/services/integrations/GcpPromQlService.ts index 24e6fade41..a36d9c2901 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.ts @@ -25,6 +25,7 @@ import { IntegrationsNotConnectedError, IntegrationsNotFoundError, IntegrationsPersistenceError, + IntegrationsUpstreamError, IntegrationsValidationError, } from "@maple/domain/http" import { GcpProjectId, type OrgId } from "@maple/domain/primitives" @@ -48,14 +49,16 @@ import { import { describeGcpApiError } from "./gcp/explain" import { mapGcpResource } from "./gcp/mapping" -/** Google's tokens live 600 s; one is reused until a minute before that. */ -const TOKEN_REUSE_MS = 9 * 60_000 +/** Google's tokens live 600 s. Three minutes to spare outlast the slowest fan-out. */ +const TOKEN_REUSE_MS = 7 * 60_000 const CONCURRENCY = 4 /** Projects a folder or organization search reads, 500 a page. */ const MAX_PROJECT_PAGES = 4 const HOUR_SECONDS = 3_600 /** A window ending this close to now still gains points: its result is kept a minute, else ten. */ const FRESH_MS = 5 * 60_000 +/** Above the default: a miss here costs a Google fan-out, not a row lookup. */ +const CACHE_READ_TIMEOUT_MS = 250 const DISTRIBUTION_SUFFIX = /_(bucket|count|sum)$/u const isProjectId = Schema.is(GcpProjectId) @@ -71,6 +74,7 @@ export type GcpPromQlError = | GcpPromQlInvalidQueryError | GcpPromQlTooManyProjectsError | GcpPromQlFailedError + | IntegrationsUpstreamError | IntegrationsPersistenceError export interface GcpPromQlServiceApi { @@ -151,11 +155,20 @@ export class GcpPromQlService extends Context.Service now) return cached.reader if (mapleToken === undefined || mapleToken.until <= now) { - const signer = yield* importServiceAccountKey(key) - mapleToken = { - token: yield* fetchMapleAccessToken(httpClient, signer, now), - until: now + TOKEN_REUSE_MS, - } + // Maple's own sign-in: its failure is no connection's, and its cause is not the customer's. + const token = yield* importServiceAccountKey(key).pipe( + Effect.flatMap((signer) => fetchMapleAccessToken(httpClient, signer, now)), + Effect.tapError((error) => + Effect.logError("gcp sign-in as Maple failed", { error: error.message }), + ), + Effect.mapError( + () => + new IntegrationsUpstreamError({ + message: "Maple could not sign in to Google Cloud", + }), + ), + ) + mapleToken = { token, until: now + TOKEN_REUSE_MS } } const reader = yield* impersonateReader(httpClient, mapleToken.token, connector) readers.set(connector.id, { reader, until: now + TOKEN_REUSE_MS }) @@ -214,7 +227,8 @@ export class GcpPromQlService extends Context.Service() + const signedIn = new Map() const failures: Array = [] for (const connector of new Set(selected.map(([, connector]) => connector))) { const reader = yield* readerFor(key, connector).pipe( @@ -302,11 +317,11 @@ export class GcpPromQlService extends Context.Service { - const reader = readers.get(connector) + const reader = signedIn.get(connector) return reader === undefined ? [] : [{ projectId, connector, reader }] }), ({ projectId, connector, reader }) => @@ -329,13 +344,24 @@ export class GcpPromQlService extends Context.Service - series.map((one) => ({ - labels: { ...one.labels, project_id: one.labels.project_id ?? projectId }, - points: one.points, - })), - ), + series: [ + // Identical label sets from two projects stay apart by `project_id`. A + // project whose metrics scope holds another answers for both: kept once. + ...new Map( + answered.flatMap(({ projectId, series }) => + series.map((one) => { + const labels = { + ...one.labels, + project_id: one.labels.project_id ?? projectId, + } + return [ + JSON.stringify(Object.entries(labels).sort()), + { labels, points: one.points }, + ] as const + }), + ), + ).values(), + ], projects: answered.map(({ projectId }) => projectId), failures, }) @@ -348,6 +374,7 @@ export class GcpPromQlService extends Context.Service result.failures.length > 0 || endMs > now - FRESH_MS ? 60 : 600, schema: GcpPromQlResult, + readTimeoutMs: CACHE_READ_TIMEOUT_MS, }, compute, ) @@ -397,7 +424,8 @@ export class GcpPromQlService extends Context.Service nothingRead([failure(projectId, connector, error)]), + failed: (error: GcpApiError) => + Effect.fail(nothingRead([failure(projectId, connector, error)])), } }) @@ -413,6 +441,7 @@ export class GcpPromQlService extends Context.Service getMetricDescriptor(httpClient, reader, target.projectId, type), ), - Effect.mapError(target.failed), + Effect.catchTag("@maple/api/integrations/GcpApiError", target.failed), ) const descriptor = (yield* find(request.type)) ?? diff --git a/packages/backend/src/services/integrations/gcp/api.ts b/packages/backend/src/services/integrations/gcp/api.ts index 2327c65a03..7ee4678a49 100644 --- a/packages/backend/src/services/integrations/gcp/api.ts +++ b/packages/backend/src/services/integrations/gcp/api.ts @@ -435,9 +435,12 @@ const decodePromQlError = Schema.decodeUnknownOption( Schema.Struct({ status: Schema.Literal("error"), errorType: Schema.String, error: Schema.String }), ) -/** Google's answer to a query it cannot run: `bad_data`, `invalid_argument` or `execution`. */ +/** + * Google's answer to a query it cannot parse or plan (400 `bad_data`). A 422 `execution` error, + * a timeout or too many samples, is the project's alone and stays a `GcpApiError`. + */ const refuseQuery = (projectId: GcpProjectId) => (status: number, body: unknown) => { - if (status !== 400 && status !== 422) return undefined + if (status !== 400) return undefined return Option.match(decodePromQlError(body), { onNone: () => undefined, onSome: ({ error }) => new GcpPromQlInvalidQueryError({ message: error.slice(0, 1_000), projectId }), @@ -458,9 +461,10 @@ export const promQlQuery = ( call( httpClient, "Cloud Monitoring", - HttpClientRequest.get(promQlUrl(projectId, "time" in params ? "query" : "query_range")).pipe( + // A form body: a 4,000-character query would make a long URL. + HttpClientRequest.post(promQlUrl(projectId, "time" in params ? "query" : "query_range")).pipe( asReader(reader), - HttpClientRequest.setUrlParams({ ...params, timeout: PROMQL_TIMEOUT }), + HttpClientRequest.bodyUrlParams({ ...params, timeout: PROMQL_TIMEOUT }), ), decodePromQlResult, refuseQuery(projectId), @@ -487,7 +491,8 @@ const decodeNames = Schema.decodeUnknownEffect(Schema.Struct({ data: Schema.Arra /** * Every metric type the project has a descriptor for, data or not. Each comes twice, in PromQL's - * UTF-8 form and the legacy `run_googleapis_com:request_count` form; only the first is kept. + * UTF-8 form and the legacy `run_googleapis_com:request_count` form (the first `/` a `:`, every + * other character outside `[A-Za-z0-9_]` a `_`); only the first is kept. */ export const listPromQlMetricNames = ( httpClient: HttpClient.HttpClient, @@ -499,7 +504,16 @@ export const listPromQlMetricNames = ( "Cloud Monitoring", HttpClientRequest.get(promQlUrl(projectId, "label/__name__/values")).pipe(asReader(reader)), decodeNames, - ).pipe(Effect.map(({ data }) => data.filter((name) => !name.includes(":")))) + ).pipe( + Effect.map(({ data }) => { + const legacy = new Set( + data + .filter((name) => name.includes("/")) + .map((name) => name.replace("/", ":").replace(/[^\w:]/gu, "_")), + ) + return data.filter((name) => !legacy.has(name)) + }), + ) const decodeMetricDescriptors = Schema.decodeUnknownEffect( Schema.Struct({ diff --git a/packages/backend/src/services/integrations/gcp/explain.ts b/packages/backend/src/services/integrations/gcp/explain.ts index e75c23c2c4..5f398e05e8 100644 --- a/packages/backend/src/services/integrations/gcp/explain.ts +++ b/packages/backend/src/services/integrations/gcp/explain.ts @@ -1,7 +1,7 @@ import type { GcpConnectorRow } from "@maple/db/tables" import type { GcpApiError } from "./api" -const RETRIES = "Maple retries in 5 minutes." +export const RETRIES = "Maple retries in 5 minutes." type ExplainedConnector = Pick< GcpConnectorRow, @@ -12,7 +12,7 @@ type ExplainedConnector = Pick< * Why a failed call failed and what to do about it, in the customer's terms. `live` is a query a * person is waiting on, which nothing retries; otherwise it is the poller's. */ -export const explainGcpApiError = (error: GcpApiError, connector: ExplainedConnector, live = false) => { +const explainGcpApiError = (error: GcpApiError, connector: ExplainedConnector, live = false) => { // The host project is the one Google checks for billing, enabled APIs and quota. A project // scope is its own host, and reads as "the project". const host = `${connector.scopeType === "project" ? "project" : "host project"} ${connector.projectId}` diff --git a/packages/domain/src/http/gcp-promql.test.ts b/packages/domain/src/http/gcp-promql.test.ts index 6b7b8b149c..603670dcb6 100644 --- a/packages/domain/src/http/gcp-promql.test.ts +++ b/packages/domain/src/http/gcp-promql.test.ts @@ -20,6 +20,8 @@ describe("GcpPromQlRangeRequest", () => { ["a query over 4,000 characters", { query: `sum(${"x".repeat(4_000)})` }], ["a step under a minute", { stepSeconds: 30 }], ["a fractional step", { stepSeconds: 90.5 }], + ["a step over a day", { stepSeconds: 86_401 }], + ["more than 11,000 points a series", { startTime: "2026-09-01 00:00:00" }], ["an inverted window", { startTime: valid.endTime, endTime: valid.startTime }], ["an impossible date", { endTime: "2026-13-45 99:99:99" }], ["a project id that is not one", { projects: ["../../organizations/1"] }], @@ -33,7 +35,7 @@ describe("GcpPromQlInstantRequest", () => { it("refuses an impossible time", () => { const decode = Schema.decodeUnknownExit(GcpPromQlInstantRequest) expect(decode({ query: "up", time: "2026-10-15 12:00:00" })._tag).toBe("Success") - expect(decode({ query: "up", time: "2026-02-30 25:00:00" })._tag).toBe("Failure") + expect(decode({ query: "up", time: "2026-10-15 25:00:00" })._tag).toBe("Failure") }) }) diff --git a/packages/domain/src/http/gcp-promql.ts b/packages/domain/src/http/gcp-promql.ts index 0a3785ba52..8c816f3ba3 100644 --- a/packages/domain/src/http/gcp-promql.ts +++ b/packages/domain/src/http/gcp-promql.ts @@ -13,6 +13,7 @@ import { BucketSeconds } from "./query-engine" export const GCP_PROMQL_MAX_PROJECTS = 20 export const GCP_PROMQL_MAX_QUERY_LENGTH = 4_000 export const GCP_PROMQL_MIN_STEP_SECONDS = 60 +export const GCP_PROMQL_MAX_POINTS = 11_000 export const GCP_METRIC_SEARCH_MAX_LIMIT = 200 /** `TinybirdDateTime` is UTC without a zone marker. */ @@ -30,21 +31,36 @@ const PromQlProjects = Schema.Array(GcpProjectId).check( Schema.isMaxLength(GCP_PROMQL_MAX_PROJECTS), ) +/** The pattern admits `2026-13-45 99:99:99`, which parses to NaN. */ +const PromQlTime = TinybirdDateTime.check( + Schema.makeFilter((time: string) => !Number.isNaN(gcpPromQlTimeMs(time)) || "must be a valid datetime"), +) + export class GcpPromQlRangeRequest extends Schema.Class("GcpPromQlRangeRequest")( Schema.Struct({ query: PromQlQuery, - startTime: TinybirdDateTime, - endTime: TinybirdDateTime, + startTime: PromQlTime, + endTime: PromQlTime, /** Point spacing. Start and end are aligned to it. */ - stepSeconds: BucketSeconds.check(Schema.isGreaterThanOrEqualTo(GCP_PROMQL_MIN_STEP_SECONDS)), + stepSeconds: BucketSeconds.check( + Schema.isGreaterThanOrEqualTo(GCP_PROMQL_MIN_STEP_SECONDS), + Schema.isLessThanOrEqualTo(86_400), + ), projects: Schema.optionalKey(PromQlProjects), }).check( Schema.makeFilter( - (request: { readonly startTime: string; readonly endTime: string }) => { + (request: { + readonly startTime: string + readonly endTime: string + readonly stepSeconds: number + }) => { const extentMs = gcpPromQlTimeMs(request.endTime) - gcpPromQlTimeMs(request.startTime) - // The pattern admits `2026-13-45 99:99:99`, which parses to NaN. - if (Number.isNaN(extentMs)) return "startTime and endTime must be valid datetimes" - return extentMs > 0 || "endTime must be after startTime" + if (extentMs <= 0) return "endTime must be after startTime" + // Google's limit per series, refused here as a request error. + return ( + extentMs / 1000 / request.stepSeconds <= GCP_PROMQL_MAX_POINTS || + `at most ${GCP_PROMQL_MAX_POINTS} points per series: widen stepSeconds` + ) }, { identifier: "GcpPromQlWindow" }, ), @@ -54,11 +70,7 @@ export class GcpPromQlRangeRequest extends Schema.Class(" export class GcpPromQlInstantRequest extends Schema.Class("GcpPromQlInstantRequest")( { query: PromQlQuery, - time: TinybirdDateTime.check( - Schema.makeFilter( - (time: string) => !Number.isNaN(gcpPromQlTimeMs(time)) || "time must be a valid datetime", - ), - ), + time: PromQlTime, projects: Schema.optionalKey(PromQlProjects), }, ) {} From fa70a303bd774aebd2018cad3428381e577102b8 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 01:25:21 +0200 Subject: [PATCH 4/8] feat(gcp): refuse PromQL answers of more than 500 series --- .../integrations/GcpPromQlService.test.ts | 29 ++++++++++++ .../services/integrations/GcpPromQlService.ts | 46 +++++++++++-------- .../anticipated-error-identifiers.ts | 1 + packages/domain/src/http/gcp-promql.ts | 17 +++++++ packages/domain/src/http/integrations.ts | 2 + 5 files changed, 77 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/services/integrations/GcpPromQlService.test.ts b/packages/backend/src/services/integrations/GcpPromQlService.test.ts index 425b9d4384..78bd8bb4ed 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.test.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.test.ts @@ -548,6 +548,35 @@ describe("GcpPromQlService.queryRange", () => { ) }) + it.effect("refuses an answer of more than 500 series across its projects, never cut short", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + // 250 series from each project: 750 in all, 500 from the folder alone. + promql: (call) => + matrix( + ...Array.from({ length: 250 }, (_, index) => ({ + metric: { instance: `vm-${index}`, project_id: call.project }, + values: [[T0, "1"]] as Array<[number, string]>, + })), + ), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const error = yield* Effect.flip(queryRange(range("up"))) + assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlTooManySeriesError") + assert.include(error.message, "750 series") + assert.include(error.message, "sum by") + + const atTheCap = yield* queryRange(range("up", ["acme-shop", "acme-data"])) + assert.lengthOf(atTheCap.series, 500) + }), + ) + }) + it.effect("signs in once and reuses the tokens for later queries", () => { const testDb = createTestDb(trackedDbs) const calls = makeCalls() diff --git a/packages/backend/src/services/integrations/GcpPromQlService.ts b/packages/backend/src/services/integrations/GcpPromQlService.ts index a36d9c2901..bdc275c7e4 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.ts @@ -20,6 +20,8 @@ import { type GcpPromQlRangeRequest, GcpPromQlResult, GcpPromQlTooManyProjectsError, + GcpPromQlTooManySeriesError, + GCP_PROMQL_MAX_SERIES, gcpPromQlTimeMs, GcpPromQlUnavailableError, IntegrationsNotConnectedError, @@ -73,6 +75,7 @@ export type GcpPromQlError = | IntegrationsValidationError | GcpPromQlInvalidQueryError | GcpPromQlTooManyProjectsError + | GcpPromQlTooManySeriesError | GcpPromQlFailedError | IntegrationsUpstreamError | IntegrationsPersistenceError @@ -342,26 +345,33 @@ export class GcpPromQlService extends Context.Service ("series" in answer ? [answer] : [])) // Not cached: a fixed setup is read again on the next try. if (answered.length === 0) return yield* nothingRead([...targets.failures, ...failures]) + // Identical label sets from two projects stay apart by `project_id`. A project + // whose metrics scope holds another answers for both: kept once. + const series = [ + ...new Map( + answered.flatMap(({ projectId, series }) => + series.map((one) => { + const labels = { + ...one.labels, + project_id: one.labels.project_id ?? projectId, + } + return [ + JSON.stringify(Object.entries(labels).sort()), + { labels, points: one.points }, + ] as const + }), + ), + ).values(), + ] + if (series.length > GCP_PROMQL_MAX_SERIES) { + return yield* new GcpPromQlTooManySeriesError({ + message: `The query returned ${series.length} series, and one answer holds at most ${GCP_PROMQL_MAX_SERIES}. Aggregate them, with sum by (…) or avg by (…), or narrow the selector.`, + series: series.length, + }) + } return new GcpPromQlResult({ resultType, - series: [ - // Identical label sets from two projects stay apart by `project_id`. A - // project whose metrics scope holds another answers for both: kept once. - ...new Map( - answered.flatMap(({ projectId, series }) => - series.map((one) => { - const labels = { - ...one.labels, - project_id: one.labels.project_id ?? projectId, - } - return [ - JSON.stringify(Object.entries(labels).sort()), - { labels, points: one.points }, - ] as const - }), - ), - ).values(), - ], + series, projects: answered.map(({ projectId }) => projectId), failures, }) diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index c3233f8547..cee74e35e5 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -51,6 +51,7 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/GcpPromQlFailedError", "@maple/http/errors/GcpPromQlInvalidQueryError", "@maple/http/errors/GcpPromQlTooManyProjectsError", + "@maple/http/errors/GcpPromQlTooManySeriesError", "@maple/http/errors/GcpPromQlUnavailableError", "@maple/http/errors/GcpScopeAlreadyConnectedError", "@maple/http/errors/IncidentTriageUnauthorizedError", diff --git a/packages/domain/src/http/gcp-promql.ts b/packages/domain/src/http/gcp-promql.ts index 8c816f3ba3..2f4e36a29c 100644 --- a/packages/domain/src/http/gcp-promql.ts +++ b/packages/domain/src/http/gcp-promql.ts @@ -14,6 +14,8 @@ export const GCP_PROMQL_MAX_PROJECTS = 20 export const GCP_PROMQL_MAX_QUERY_LENGTH = 4_000 export const GCP_PROMQL_MIN_STEP_SECONDS = 60 export const GCP_PROMQL_MAX_POINTS = 11_000 +/** Series in one answer, across its projects. */ +export const GCP_PROMQL_MAX_SERIES = 500 export const GCP_METRIC_SEARCH_MAX_LIMIT = 200 /** `TinybirdDateTime` is UTC without a zone marker. */ @@ -156,6 +158,21 @@ export class GcpPromQlTooManyProjectsError extends HttpTaggedError()( + "@maple/http/errors/GcpPromQlTooManySeriesError", + { message: Schema.String, series: Schema.Number }, + { + status: 400, + code: "gcp_promql_too_many_series", + title: "Too many series", + param: "query", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + /** Nothing could be read: every project or connection failed. */ export class GcpPromQlFailedError extends HttpTaggedError()( "@maple/http/errors/GcpPromQlFailedError", diff --git a/packages/domain/src/http/integrations.ts b/packages/domain/src/http/integrations.ts index a4031d2193..9a16130eee 100644 --- a/packages/domain/src/http/integrations.ts +++ b/packages/domain/src/http/integrations.ts @@ -21,6 +21,7 @@ import { GcpPromQlRangeRequest, GcpPromQlResult, GcpPromQlTooManyProjectsError, + GcpPromQlTooManySeriesError, GcpPromQlUnavailableError, } from "./gcp-promql" import { PrReviewOrgSettings, PrReviewRepositoryConfig } from "./pr-review" @@ -1119,6 +1120,7 @@ const GCP_PROMQL_ERRORS = [ IntegrationsValidationError, GcpPromQlInvalidQueryError, GcpPromQlTooManyProjectsError, + GcpPromQlTooManySeriesError, GcpPromQlFailedError, IntegrationsUpstreamError, IntegrationsPersistenceError, From 767f1284ffd21c1291f74fde691170d1ed03a71f Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 11:53:37 +0200 Subject: [PATCH 5/8] fix(gcp): live PromQL reads only set-up connections, once per metrics scope, within a byte cap --- .../integrations/GcpPromQlService.test.ts | 297 +++++++++++++++--- .../services/integrations/GcpPromQlService.ts | 201 +++++++++--- .../src/services/integrations/gcp/api.ts | 158 +++++++--- .../src/services/integrations/gcp/explain.ts | 14 +- packages/domain/src/http/gcp-promql.test.ts | 9 + packages/domain/src/http/gcp-promql.ts | 46 ++- 6 files changed, 593 insertions(+), 132 deletions(-) diff --git a/packages/backend/src/services/integrations/GcpPromQlService.test.ts b/packages/backend/src/services/integrations/GcpPromQlService.test.ts index 78bd8bb4ed..19bc08fd26 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.test.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.test.ts @@ -70,6 +70,7 @@ interface Calls { readonly promql: Array readonly descriptors: Array readonly assets: Array + readonly scopes: Array } interface StubOptions { @@ -80,8 +81,29 @@ interface StubOptions { readonly assets?: () => Response | undefined readonly names?: ReadonlyArray readonly descriptors?: Record + /** Projects a project's metrics scope reads besides itself. */ + readonly scopes?: Record> } +/** A stand-in project number: Google names a metrics scope's projects by number. */ +const numberOf = (projectId: string) => + String([...projectId].reduce((hash, char) => (hash * 31 + char.charCodeAt(0)) % 1_000_000_007, 7)) + +const googleError = (status: number, code: string, reason?: string) => + json( + { + error: { + code: status, + status: code, + message: "refused", + ...(reason === undefined + ? undefined + : { details: [{ "@type": "type.googleapis.com/google.rpc.ErrorInfo", reason }] }), + }, + }, + status, + ) + const matrix = ( ...result: ReadonlyArray<{ metric: Record; values: Array<[number, string]> }> ) => json({ status: "success", data: { resultType: "matrix", result } }) @@ -139,6 +161,17 @@ const stubFetch = (calls: Calls, options: StubOptions = {}) => return json({ status: "success", data: options.names ?? [] }) return options.promql?.(call) ?? matrix() } + const scope = /^\/v1\/locations\/global\/metricsScopes\/(.+)$/.exec(url.pathname) + if (url.host === "monitoring.googleapis.com" && scope !== null) { + const projectId = scope[1]! + calls.scopes.push(projectId) + return json({ + name: `locations/global/metricsScopes/${numberOf(projectId)}`, + monitoredProjects: [projectId, ...(options.scopes?.[projectId] ?? [])].map((monitored) => ({ + name: `locations/global/metricsScopes/${numberOf(projectId)}/projects/${numberOf(monitored)}`, + })), + }) + } if (url.host === "monitoring.googleapis.com" && url.pathname.endsWith("/metricDescriptors")) { const type = /metric\.type = "([^"]+)"/.exec(url.searchParams.get("filter") ?? "")?.[1] ?? "" calls.descriptors.push(type) @@ -148,7 +181,14 @@ const stubFetch = (calls: Calls, options: StubOptions = {}) => throw new Error(`unexpected request to ${url.href}`) }) as typeof fetch -const makeCalls = (): Calls => ({ oauth: 0, impersonate: [], promql: [], descriptors: [], assets: [] }) +const makeCalls = (): Calls => ({ + oauth: 0, + impersonate: [], + promql: [], + descriptors: [], + assets: [], + scopes: [], +}) const run = ( testDb: TestDb, @@ -195,7 +235,8 @@ const insertConnector = ( readonly orgId?: string readonly projectId?: string readonly folder?: string - readonly metricsEnabled?: boolean + /** What the setup script reported: the read-only account is in place. */ + readonly accountInPlace?: boolean | null } = {}, ) => { const projectId = options.projectId ?? "acme-prod" @@ -204,7 +245,7 @@ const insertConnector = ( testDb, `INSERT INTO gcp_connectors (id, org_id, scope_type, scope_id, project_id, secret_ciphertext, secret_iv, secret_tag, - secret_hash, metrics_enabled, created_by, created_at, updated_at) + secret_hash, applied_metrics_enabled, created_by, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, 'c', 'i', 't', $1, $6, 'user_1', now(), now())`, [ id, @@ -212,7 +253,7 @@ const insertConnector = ( options.folder === undefined ? "project" : "folder", options.folder ?? projectId, projectId, - options.metricsEnabled ?? false, + options.accountInPlace === undefined ? true : options.accountInPlace, ], ), ) @@ -222,11 +263,7 @@ const insertConnector = ( const insertConnectors = (testDb: TestDb) => Effect.all([ insertConnector(testDb, PROJECT_CONNECTOR), - insertConnector(testDb, FOLDER_CONNECTOR, { - projectId: "acme-host", - folder: "123", - metricsEnabled: true, - }), + insertConnector(testDb, FOLDER_CONNECTOR, { projectId: "acme-host", folder: "123" }), insertConnector(testDb, OTHER_CONNECTOR, { orgId: OTHER_ORG, projectId: "rival-prod" }), ]) @@ -270,26 +307,15 @@ describe("GcpPromQlService.queryRange", () => { [T0 + 1800, "1.5"], ], }) - : // A project's metrics scope can answer for another project: Google's label stays, - // and acme-data, whose scope holds acme-shop, answers acme-shop's series again. - matrix( - { - metric: { - service: "checkout", - project_id: - call.project === "acme-shop" ? "acme-monitored" : call.project, - }, - values: [[T0, "7"]], + : // Google's own `project_id` stays. + matrix({ + metric: { + service: "checkout", + project_id: + call.project === "acme-shop" ? "acme-monitored" : call.project, }, - ...(call.project === "acme-data" - ? [ - { - metric: { service: "checkout", project_id: "acme-monitored" }, - values: [[T0, "7"]] as Array<[number, string]>, - }, - ] - : []), - ), + values: [[T0, "7"]], + }), }), Effect.gen(function* () { yield* insertConnectors(testDb) @@ -472,8 +498,26 @@ describe("GcpPromQlService.queryRange", () => { stubFetch(calls, { promql: (call) => call.params.query === "slow" - ? json({ status: "error", errorType: "execution", error: "query timed out" }, 422) - : json({ status: "error", errorType: "bad_data", error: refusal }, 400), + ? json( + { + status: "error", + errorType: "execution", + error: "query processing would load too many samples", + }, + 422, + ) + : call.params.query === "slower" + ? json( + { + status: "error", + errorType: "timeout", + error: "query timed out in expression evaluation", + }, + 503, + ) + : call.params.query === "busy" + ? googleError(429, "RESOURCE_EXHAUSTED") + : json({ status: "error", errorType: "bad_data", error: refusal }, 400), }), Effect.gen(function* () { yield* insertConnector(testDb, PROJECT_CONNECTOR) @@ -487,9 +531,28 @@ describe("GcpPromQlService.queryRange", () => { "acme-prod", ) - // A query that ran out of time is the project's failure, not the query's fault. - const slow = yield* Effect.flip(queryRange(range("slow"))) - assert.strictEqual(slow._tag, "@maple/http/errors/GcpPromQlFailedError") + // Too many samples, out of time, or past Google's size limit: the project's failure, + // with what to change about the query. + const failureOf = (query: string) => + Effect.flip(queryRange(range(query))).pipe( + Effect.map((error) => + error._tag === "@maple/http/errors/GcpPromQlFailedError" + ? error.failures[0]!.message + : error._tag, + ), + ) + assert.strictEqual( + yield* failureOf("slow"), + "Google could not finish the query on this project. Narrow the query or widen the step. (Cloud Monitoring returned 422 execution: query processing would load too many samples)", + ) + assert.strictEqual( + yield* failureOf("slower"), + "Google could not finish the query on this project. Narrow the query or widen the step. (Cloud Monitoring returned 503 timeout: query timed out in expression evaluation)", + ) + assert.strictEqual( + yield* failureOf("busy"), + "Google refused the read for the project acme-prod: a rate limit, or an answer past Google's size limit. Narrow the query, or try again in a few minutes. (Cloud Monitoring returned 429 RESOURCE_EXHAUSTED)", + ) }), ) }) @@ -516,6 +579,16 @@ describe("GcpPromQlService.queryRange", () => { } // The second answer came from the cache. assert.lengthOf(calls.promql, 1) + + // A project the failed listing may hold is not refused as unknown. + const error = yield* Effect.flip(queryRange(range("up", ["acme-shop"]))) + assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlFailedError") + assert.deepStrictEqual( + error._tag === "@maple/http/errors/GcpPromQlFailedError" + ? error.failures.map((failure) => failure.scope) + : [], + ["folders/123"], + ) }), ) }) @@ -568,8 +641,10 @@ describe("GcpPromQlService.queryRange", () => { yield* insertConnectors(testDb) const error = yield* Effect.flip(queryRange(range("up"))) assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlTooManySeriesError") - assert.include(error.message, "750 series") - assert.include(error.message, "sum by") + assert.strictEqual( + error.message, + "The query returned more than 500 series across its projects. Aggregate them with sum by (…) or avg by (…), or narrow the selector.", + ) const atTheCap = yield* queryRange(range("up", ["acme-shop", "acme-data"])) assert.lengthOf(atTheCap.series, 500) @@ -577,6 +652,126 @@ describe("GcpPromQlService.queryRange", () => { ) }) + it.effect("refuses a project's answer past 8 MB without reading the rest", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + promql: () => matrix({ metric: { pod: "x".repeat(9 * 1024 * 1024) }, values: [[T0, "1"]] }), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(queryRange(range("up"))) + assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlTooManySeriesError") + assert.strictEqual( + error.message, + "Google's answer for project acme-prod is larger than 8 MB: too many series or points. Aggregate with sum by (…) or avg by (…), narrow the selector, or widen the step.", + ) + }), + ) + }) + + it.effect("reads a project another target's metrics scope holds through that scope, once", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + scopes: { "acme-data": ["acme-shop"] }, + // What Google answers for `sum by (service)`: no `project_id`. + promql: (call) => + matrix({ + metric: { service: "checkout" }, + values: [[T0, call.project === "acme-data" ? "9" : "3"]], + }), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const result = yield* queryRange(range("sum by (service) (rate(x[5m]))")) + assert.sameMembers( + calls.promql.map((call) => call.project), + ["acme-data", "acme-prod"], + ) + assert.deepStrictEqual>(result.projects, [ + "acme-data", + "acme-prod", + "acme-shop", + ]) + // acme-data's sum covers acme-shop too, so it names the scope, not a project. + assert.sameDeepMembers( + result.series.map((series) => series.labels), + [ + { service: "checkout", metrics_scope: "acme-data" }, + { service: "checkout", project_id: "acme-prod" }, + ], + ) + }), + ) + }) + + it.effect("reads only connections whose read-only account is in place", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const other = Schema.decodeSync(OrgId)(OTHER_ORG) + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + promql: (call) => matrix({ metric: { project_id: call.project }, values: [[T0, "1"]] }), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + yield* insertConnector(testDb, FOLDER_CONNECTOR, { + projectId: "acme-host", + folder: "123", + accountInPlace: null, + }) + yield* insertConnector(testDb, OTHER_CONNECTOR, { + orgId: OTHER_ORG, + projectId: "rival-prod", + accountInPlace: false, + }) + // The folder is left out without a word while another connection reads. + const result = yield* queryRange(range("up")) + assert.deepStrictEqual>(result.projects, ["acme-prod"]) + assert.deepStrictEqual(result.failures, []) + assert.deepStrictEqual(calls.assets, []) + assert.deepStrictEqual(calls.impersonate, [PROJECT_READER]) + + // With none in place, each connection says what to do. + const error = yield* Effect.flip( + GcpPromQlService.use((gcp) => gcp.queryRange(other, range("up"))), + ) + assert.strictEqual(error._tag, "@maple/http/errors/GcpPromQlFailedError") + assert.deepStrictEqual( + error._tag === "@maple/http/errors/GcpPromQlFailedError" + ? error.failures.map((failure) => failure.scope) + : [], + ["projects/rival-prod"], + ) + assert.include(error.message, "Run the setup script again: it creates the account.") + }), + ) + }) + + it.effect("fails as Maple's own when IAM refuses Maple's project, not as the connection's", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + impersonate: () => googleError(403, "PERMISSION_DENIED", "SERVICE_DISABLED"), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(queryRange(range("up"))) + assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsUpstreamError") + }), + ) + }) + it.effect("signs in once and reuses the tokens for later queries", () => { const testDb = createTestDb(trackedDbs) const calls = makeCalls() @@ -639,7 +834,21 @@ describe("GcpPromQlService.queryInstant", () => { return run( testDb, stubFetch(calls, { - promql: () => json({ status: "success", data: { resultType: "scalar", result: [T0, "2"] } }), + promql: (call) => + call.params.query === "x[5m]" + ? matrix({ + metric: { project_id: "acme-prod" }, + values: [ + [T0, "1"], + [T0 + 60, "2"], + ], + }) + : call.params.query === '"text"' + ? json({ + status: "success", + data: { resultType: "string", result: [T0, "text"] }, + }) + : json({ status: "success", data: { resultType: "scalar", result: [T0, "2"] } }), }), Effect.gen(function* () { yield* insertConnector(testDb, PROJECT_CONNECTOR) @@ -655,6 +864,22 @@ describe("GcpPromQlService.queryInstant", () => { assert.deepStrictEqual(result.series, [ { labels: { project_id: "acme-prod" }, points: [[T0, 2]] }, ]) + + // A range selector answers a matrix, as Google says. + const instant = (query: string) => + GcpPromQlService.use((gcp) => + gcp.queryInstant( + ORG, + new GcpPromQlInstantRequest({ query, time: "2026-10-15 11:00:42" }), + ), + ) + const window = yield* instant("x[5m]") + assert.strictEqual(window.resultType, "matrix") + assert.lengthOf(window.series[0]!.points, 2) + + const text = yield* Effect.flip(instant('"text"')) + assert.strictEqual(text._tag, "@maple/http/errors/GcpPromQlInvalidQueryError") + assert.include(text.message, "String results are not supported") }), ) }) diff --git a/packages/backend/src/services/integrations/GcpPromQlService.ts b/packages/backend/src/services/integrations/GcpPromQlService.ts index bdc275c7e4..219b0e8c8f 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.ts @@ -1,9 +1,9 @@ /** * Live PromQL over the Google Cloud projects an organization has connected, through Google's - * Prometheus-compatible API. Every connection is a target, whatever its metrics switch: a project - * scope is its project, a folder or organization the active projects Cloud Asset finds under it. - * Each project is read with its connection's read-only account, one call per project, and what - * fails is reported beside what answered. + * Prometheus-compatible API. A connection whose read-only account is in place is a target: a + * project scope is its project, a folder or organization the active projects Cloud Asset finds + * under it. Each project is read with its connection's read-only account, one call per metrics + * scope, and what fails is reported beside what answered. */ import * as PG from "@maple-dev/effect-orm/postgres" import { EdgeCacheService } from "@maple/cache" @@ -22,6 +22,7 @@ import { GcpPromQlTooManyProjectsError, GcpPromQlTooManySeriesError, GCP_PROMQL_MAX_SERIES, + gcpPromQlRange, gcpPromQlTimeMs, GcpPromQlUnavailableError, IntegrationsNotConnectedError, @@ -39,6 +40,7 @@ import { Env } from "@maple/backend/platform/Env" import { fetchMapleAccessToken, getMetricDescriptor, + getMetricsScope, type GcpApiError, type GcpPromQlParams, type GcpReader, @@ -61,6 +63,8 @@ const HOUR_SECONDS = 3_600 const FRESH_MS = 5 * 60_000 /** Above the default: a miss here costs a Google fan-out, not a row lookup. */ const CACHE_READ_TIMEOUT_MS = 250 +/** Why IAM Credentials refuses Maple's own project; impersonation does not go through the host. */ +const MAPLE_SIDE_REASONS = new Set(["SERVICE_DISABLED", "BILLING_DISABLED", "USER_PROJECT_DENIED"]) const DISTRIBUTION_SUFFIX = /_(bucket|count|sum)$/u const isProjectId = Schema.is(GcpProjectId) @@ -69,6 +73,12 @@ type Connector = Pick `${connector.scopeType}s/${connector.scopeId}` +/** A project's metrics scope: its number, and the numbers of the projects it reads, itself included. */ +const MetricsScope = Schema.Struct({ number: Schema.String, monitored: Schema.Array(Schema.String) }) + +const NOT_SET_UP = + "This connection's read-only account is not in place yet, so live queries can't read through it. Run the setup script again: it creates the account. If the script stops at an organization policy that restricts sharing to your domain, follow the setup guide's section on domain-restricted sharing." + export type GcpPromQlError = | GcpPromQlUnavailableError | IntegrationsNotConnectedError @@ -173,7 +183,22 @@ export class GcpPromQlService extends Context.Service + Effect.gen(function* () { + if (error.kind !== "upstream" && !MAPLE_SIDE_REASONS.has(error.reason ?? "")) + return yield* error + yield* Effect.logError("gcp impersonation failed on Maple's side", { + error: error.message, + }) + return yield* new IntegrationsUpstreamError({ + message: "Maple could not sign in to Google Cloud", + }) + }), + ), + ) readers.set(connector.id, { reader, until: now + TOKEN_REUSE_MS }) return reader }) @@ -211,7 +236,14 @@ export class GcpPromQlService extends Context.Service db.run( PG.from(GcpConnectors) - .select("id", "scopeType", "scopeId", "projectId", "lastMetricsReceivedAt") + .select( + "id", + "scopeType", + "scopeId", + "projectId", + "lastMetricsReceivedAt", + "appliedMetricsEnabled", + ) .where(($) => [$.orgId.eq(orgId)]) .orderBy(($) => [ [$.createdAt, "asc"], @@ -225,8 +257,16 @@ export class GcpPromQlService extends Context.Service connector.appliedMetricsEnabled === true) + if (ready.length === 0) { + return yield* nothingRead( + connectors.map((connector) => ({ scope: scopeName(connector), message: NOT_SET_UP })), + ) + } // One connection at a time, so a cold isolate signs in as Maple once. - const listed = yield* Effect.forEach(connectors, (connector) => + const listed = yield* Effect.forEach(ready, (connector) => connector.scopeType === "project" ? Effect.succeed({ connector, @@ -289,7 +329,11 @@ export class GcpPromQlService extends Context.Service !targets.projects.has(projectId)) - if (unknown.length > 0) return yield* notReadable(unknown) + if (unknown.length > 0) { + // A connection whose projects could not be listed may hold them. + if (targets.failures.length > 0) return yield* nothingRead(targets.failures) + return yield* notReadable(unknown) + } selected = selected.filter(([projectId]) => requested.includes(projectId)) } else if (selected.length > GCP_PROMQL_MAX_PROJECTS) { return yield* new GcpPromQlTooManyProjectsError({ @@ -297,10 +341,14 @@ export class GcpPromQlService extends Context.Service projectId), }) } - const resultType = "time" in params ? "vector" : "matrix" if (selected.length === 0) { if (targets.failures.length > 0) return yield* nothingRead(targets.failures) - return new GcpPromQlResult({ resultType, series: [], projects: [], failures: [] }) + return new GcpPromQlResult({ + resultType: "time" in params ? "vector" : "matrix", + series: [], + projects: [], + failures: [], + }) } selected.sort(([a], [b]) => a.localeCompare(b)) yield* Effect.annotateCurrentSpan({ "maple.gcp.projects": selected.length }) @@ -322,18 +370,72 @@ export class GcpPromQlService extends Context.Service { const reader = signedIn.get(connector) return reader === undefined ? [] : [{ projectId, connector, reader }] }), - ({ projectId, connector, reader }) => - promQlQuery(httpClient, reader, projectId, params).pipe( - Effect.map((series) => ({ projectId, series })), + (target) => + edgeCache + .getOrCompute( + { + bucket: "gcp-metrics-scopes", + key: `${orgId}:${target.projectId}`, + ttlSeconds: HOUR_SECONDS, + readTimeoutMs: CACHE_READ_TIMEOUT_MS, + schema: MetricsScope, + }, + getMetricsScope(httpClient, target.reader, target.projectId), + ) + .pipe( + Effect.map(({ value }) => ({ ...target, scope: value })), + Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => + Effect.succeed({ + ...target, + failure: failure(target.projectId, target.connector, error), + }), + ), + ), + { concurrency: CONCURRENCY }, + ) + failures.push( + ...scoped.flatMap((target) => ("failure" in target ? [target.failure] : [])), + ) + const withScope = scoped.flatMap((target) => ("scope" in target ? [target] : [])) + const covered = new Set() + const reads: typeof withScope = [] + // Widest scopes first. + for (const target of [...withScope].sort( + (a, b) => b.scope.monitored.length - a.scope.monitored.length, + )) { + if (covered.has(target.scope.number)) continue + for (const number of target.scope.monitored) covered.add(number) + reads.push(target) + } + + let seriesCount = 0 + const answers = yield* Effect.forEach( + reads, + (read) => + promQlQuery(httpClient, read.reader, read.projectId, params).pipe( + // Checked as each project answers, so a broad query stops early. + Effect.flatMap((answer) => + Effect.gen(function* () { + seriesCount += answer.series.length + if (seriesCount > GCP_PROMQL_MAX_SERIES) { + return yield* new GcpPromQlTooManySeriesError({ + message: `The query returned more than ${GCP_PROMQL_MAX_SERIES} series across its projects. Aggregate them with sum by (…) or avg by (…), or narrow the selector.`, + }) + } + return { read, answer } + }), + ), Effect.catchTag("@maple/api/integrations/GcpApiError", (error) => Effect.succeed({ - projectId, - failure: failure(projectId, connector, error), + read, + failure: failure(read.projectId, read.connector, error), }), ), ), @@ -342,37 +444,37 @@ export class GcpPromQlService extends Context.Service ("failure" in answer ? [answer.failure] : [])), ) - const answered = answers.flatMap((answer) => ("series" in answer ? [answer] : [])) + const answered = answers.flatMap((answer) => ("answer" in answer ? [answer] : [])) + const [first] = answered // Not cached: a fixed setup is read again on the next try. - if (answered.length === 0) return yield* nothingRead([...targets.failures, ...failures]) - // Identical label sets from two projects stay apart by `project_id`. A project - // whose metrics scope holds another answers for both: kept once. - const series = [ - ...new Map( - answered.flatMap(({ projectId, series }) => - series.map((one) => { - const labels = { - ...one.labels, - project_id: one.labels.project_id ?? projectId, - } - return [ - JSON.stringify(Object.entries(labels).sort()), - { labels, points: one.points }, - ] as const - }), - ), - ).values(), - ] - if (series.length > GCP_PROMQL_MAX_SERIES) { - return yield* new GcpPromQlTooManySeriesError({ - message: `The query returned ${series.length} series, and one answer holds at most ${GCP_PROMQL_MAX_SERIES}. Aggregate them, with sum by (…) or avg by (…), or narrow the selector.`, - series: series.length, - }) - } + if (first === undefined) return yield* nothingRead([...targets.failures, ...failures]) return new GcpPromQlResult({ - resultType, - series, - projects: answered.map(({ projectId }) => projectId), + resultType: first.answer.resultType, + series: answered.flatMap(({ read, answer }) => + answer.series.map((one) => ({ + // Identical label sets from two projects stay apart. A scope of several + // projects names itself, not a project its series may not come from. + labels: + one.labels.project_id === undefined + ? { + ...one.labels, + [read.scope.monitored.length > 1 + ? "metrics_scope" + : "project_id"]: read.projectId, + } + : one.labels, + points: one.points, + })), + ), + // A target answers through the read whose scope holds it. + projects: withScope + .filter((target) => + answered.some( + ({ read }) => + read === target || read.scope.monitored.includes(target.scope.number), + ), + ) + .map((target) => target.projectId), failures, }) }) @@ -398,10 +500,8 @@ export class GcpPromQlService extends Context.Service 0) return yield* nothingRead(targets.failures) + return yield* notReadable([project]) + } if (projectId === undefined || connector === undefined) return yield* nothingRead(targets.failures) return { diff --git a/packages/backend/src/services/integrations/gcp/api.ts b/packages/backend/src/services/integrations/gcp/api.ts index 7ee4678a49..5ffc5e52d8 100644 --- a/packages/backend/src/services/integrations/gcp/api.ts +++ b/packages/backend/src/services/integrations/gcp/api.ts @@ -4,8 +4,8 @@ * here may leak a credential: tokens stay `Redacted`, and a failure carries the HTTP status and * Google's error code and reason only, never a response body, transport error or schema issue. * The one sentence of Google's that is read, to tell a missing billing account from a missing - * role, chooses a reason and is dropped. The exception is a refused PromQL query: Google's - * explanation of the caller's own query is passed on. + * role, chooses a reason and is dropped. The exception is PromQL: Google's explanation of why + * the caller's own query was refused or did not finish is passed on. * Untraced on purpose: the HTTP client records a span per request, and "not set up yet" is not an * exception. */ @@ -16,9 +16,9 @@ import { type GcpMetric, type GcpMetricGroup, } from "@maple/domain/gcp-metrics" -import { GcpPromQlInvalidQueryError } from "@maple/domain/http" +import { GcpPromQlInvalidQueryError, GcpPromQlTooManySeriesError } from "@maple/domain/http" import type { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" -import { Duration, Effect, Option, Redacted, Schema } from "effect" +import { Duration, Effect, Option, Redacted, Schema, Stream } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" const TOKEN_URL = "https://oauth2.googleapis.com/token" @@ -118,21 +118,39 @@ const errorReason = (body: unknown, kind: GcpApiError["kind"]): string | undefin .flatMap((detail) => Option.toArray(decodeErrorInfo(detail)))[0]?.reason ?? (kind === "denied" ? reasonFromMessage(body) : undefined) +/** The largest answer read. A poller page is about 1 MB; a bigger PromQL answer is refused. */ +export const MAX_BODY_BYTES = 8 * 1024 * 1024 +const parseJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)) + /** - * One JSON call. `api` names the Google service in the error. `refuse` turns a failed answer - * into the caller's own error instead. + * One JSON call. `api` names the Google service in the error. `refuse` turns a failed answer, + * and `tooLarge` one past `MAX_BODY_BYTES`, into the caller's own error instead. */ -const call = ( +const call = ( httpClient: HttpClient.HttpClient, api: GcpApiError["api"], request: HttpClientRequest.HttpClientRequest, decode: (body: unknown) => Effect.Effect, refuse?: (status: number, body: unknown) => E | undefined, -): Effect.Effect => { + tooLarge?: () => E2, +): Effect.Effect => { const failed = (what: string) => new GcpApiError({ message: `${api} ${what}`, api, kind: "upstream" }) return Effect.gen(function* () { const response = yield* httpClient.execute(request) - const body = yield* response.json.pipe(Effect.orElseSucceed(() => null)) + // Read as it arrives, never trusting Content-Length: past the cap the rest is left unread. + const chunks: Array = [] + let size = 0 + yield* Stream.runForEachWhile(response.stream, (chunk) => + Effect.sync(() => { + size += chunk.byteLength + if (size <= MAX_BODY_BYTES) chunks.push(chunk) + return size <= MAX_BODY_BYTES + }), + ) + const body = + size > MAX_BODY_BYTES + ? undefined + : Option.getOrNull(parseJson(Buffer.concat(chunks).toString("utf8"))) return { status: response.status, body } }).pipe( Effect.timeout(REQUEST_TIMEOUT), @@ -142,6 +160,10 @@ const call = ( }), Effect.flatMap(({ status, body }) => Effect.gen(function* () { + if (body === undefined) { + if (tooLarge !== undefined) return yield* Effect.fail(tooLarge()) + return yield* failed("answered with more than 8 MB") + } if (status >= 300) { const refused = refuse?.(status, body) if (refused !== undefined) return yield* Effect.fail(refused) @@ -291,9 +313,6 @@ export const impersonateReader = Effect.fnUntraced(function* ( }), ), decodeImpersonatedToken, - ).pipe( - // Not made through the host project, so its reason says nothing about that project. - Effect.mapError(({ message, api, kind }) => new GcpApiError({ message, api, kind })), ) return { accessToken: response.accessToken, quotaProject: connector.projectId } satisfies GcpReader }) @@ -421,6 +440,7 @@ const decodePromQlResult = Schema.decodeUnknownEffect( }), // `1 + 1`, `time()`: answered as one series without labels. Schema.Struct({ resultType: Schema.Literal("scalar"), result: Sample }), + Schema.Struct({ resultType: Schema.Literal("string"), result: Schema.Unknown }), ]), }), ) @@ -436,22 +456,34 @@ const decodePromQlError = Schema.decodeUnknownOption( ) /** - * Google's answer to a query it cannot parse or plan (400 `bad_data`). A 422 `execution` error, - * a timeout or too many samples, is the project's alone and stays a `GcpApiError`. + * Google's own error for a query: 400 `bad_data` refuses the query itself. 422 `execution` and + * 503 `timeout` (too many samples, out of time) are the project's alone and stay a + * `GcpApiError`, carrying Google's text. */ -const refuseQuery = (projectId: GcpProjectId) => (status: number, body: unknown) => { - if (status !== 400) return undefined - return Option.match(decodePromQlError(body), { +const refuseQuery = (projectId: GcpProjectId) => (status: number, body: unknown) => + Option.match(decodePromQlError(body), { onNone: () => undefined, - onSome: ({ error }) => new GcpPromQlInvalidQueryError({ message: error.slice(0, 1_000), projectId }), + onSome: ({ errorType, error }) => + status === 400 + ? new GcpPromQlInvalidQueryError({ message: error.slice(0, 1_000), projectId }) + : status === 422 || status === 503 + ? new GcpApiError({ + message: `Cloud Monitoring returned ${status} ${errorType}: ${error.slice(0, 500)}`, + api: "Cloud Monitoring", + kind: status === 422 ? "invalid" : "upstream", + reason: "QUERY_EXECUTION", + }) + : undefined, }) -} export type GcpPromQlParams = | { readonly query: string; readonly time: number } | { readonly query: string; readonly start: number; readonly end: number; readonly step: number } -/** One instant or range query against one project, times in epoch seconds. */ +/** + * One instant or range query against one project, times in epoch seconds. An instant query of a + * range selector answers a matrix; a scalar is a vector of one series without labels. + */ export const promQlQuery = ( httpClient: HttpClient.HttpClient, reader: GcpReader, @@ -468,23 +500,77 @@ export const promQlQuery = ( ), decodePromQlResult, refuseQuery(projectId), + () => + new GcpPromQlTooManySeriesError({ + message: `Google's answer for project ${projectId} is larger than 8 MB: too many series or points. Aggregate with sum by (…) or avg by (…), narrow the selector, or widen the step.`, + }), ).pipe( - Effect.map(({ data }) => { - switch (data.resultType) { - case "matrix": - return data.result.map((series) => ({ - labels: series.metric, - points: toPoints(series.values), - })) - case "vector": - return data.result.map((series) => ({ - labels: series.metric, - points: toPoints([series.value]), - })) - case "scalar": - return [{ labels: {}, points: toPoints([data.result]) }] - } - }), + Effect.flatMap(({ data }) => + Effect.gen(function* () { + switch (data.resultType) { + case "matrix": + return { + resultType: "matrix" as const, + series: data.result.map((series) => ({ + labels: series.metric, + points: toPoints(series.values), + })), + } + case "vector": + return { + resultType: "vector" as const, + series: data.result.map((series) => ({ + labels: series.metric, + points: toPoints([series.value]), + })), + } + case "scalar": + return { + resultType: "vector" as const, + series: [{ labels: {}, points: toPoints([data.result]) }], + } + case "string": + return yield* new GcpPromQlInvalidQueryError({ + message: + "String results are not supported: query a vector, a range vector or a scalar.", + projectId, + }) + } + }), + ), + ) + +const decodeMetricsScope = Schema.decodeUnknownEffect( + Schema.Struct({ + name: Schema.String, + monitoredProjects: Schema.optionalKey(Schema.Array(Schema.Struct({ name: Schema.String }))), + }), +) + +const lastSegment = (name: string) => name.slice(name.lastIndexOf("/") + 1) + +/** + * The metrics scope a query on the project reads: its own project number, and the numbers of + * the projects it monitors, itself included. Needs `resourcemanager.projects.get`, which + * Monitoring Viewer holds. + */ +export const getMetricsScope = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, +) => + call( + httpClient, + "Cloud Monitoring", + HttpClientRequest.get( + `https://monitoring.googleapis.com/v1/locations/global/metricsScopes/${projectId}`, + ).pipe(asReader(reader)), + decodeMetricsScope, + ).pipe( + Effect.map((scope) => ({ + number: lastSegment(scope.name), + monitored: (scope.monitoredProjects ?? []).map((project) => lastSegment(project.name)), + })), ) const decodeNames = Schema.decodeUnknownEffect(Schema.Struct({ data: Schema.Array(Schema.String) })) diff --git a/packages/backend/src/services/integrations/gcp/explain.ts b/packages/backend/src/services/integrations/gcp/explain.ts index 5f398e05e8..6074a34a55 100644 --- a/packages/backend/src/services/integrations/gcp/explain.ts +++ b/packages/backend/src/services/integrations/gcp/explain.ts @@ -18,10 +18,16 @@ const explainGcpApiError = (error: GcpApiError, connector: ExplainedConnector, l const host = `${connector.scopeType === "project" ? "project" : "host project"} ${connector.projectId}` const scope = `${connector.scopeType} ${connector.scopeId}` const retries = live ? "Try again in a few minutes." : RETRIES - if (error.reason === "BILLING_DISABLED") { + // Signing in does not go through the host project: a reason there is about Maple's own. + const hostReason = error.api === "IAM Credentials" ? undefined : error.reason + if (hostReason === "BILLING_DISABLED") { return `The ${host} has no active billing account, and ${error.api} only answers for projects that have one. Link one: https://console.cloud.google.com/billing/linkedaccount?project=${connector.projectId}${live ? "" : " Maple retries every 5 minutes."}` } - if (error.reason === "SERVICE_DISABLED") { + // A PromQL query that ran out of samples or time; Google's text, in the status, says which. + if (error.reason === "QUERY_EXECUTION") { + return "Google could not finish the query on this project. Narrow the query or widen the step." + } + if (hostReason === "SERVICE_DISABLED") { return `The ${error.api} API is switched off in the ${host}. Run the setup script again: it switches the API on.` } if (error.kind === "denied" || error.kind === "not_found") { @@ -43,6 +49,10 @@ const explainGcpApiError = (error: GcpApiError, connector: ExplainedConnector, l } // Signing in is counted against Maple's own project, not the host project. if (error.kind === "rate_limited" && error.api !== "IAM Credentials") { + // Google answers an answer past its size limit the same way as a spent quota. + if (live) { + return `Google refused the read for the ${host}: a rate limit, or an answer past Google's size limit. Narrow the query, or try again in a few minutes.` + } return `Google rate-limited the ${error.api} API for the ${host}. ${retries} If this repeats, raise that API's quota on the project.` } return `Maple's request to Google's ${error.api} API failed. ${retries} If this repeats, write to support@maple.dev.` diff --git a/packages/domain/src/http/gcp-promql.test.ts b/packages/domain/src/http/gcp-promql.test.ts index 603670dcb6..f21582a349 100644 --- a/packages/domain/src/http/gcp-promql.test.ts +++ b/packages/domain/src/http/gcp-promql.test.ts @@ -13,6 +13,10 @@ const valid = { describe("GcpPromQlRangeRequest", () => { it("accepts a query over a window with a step of a minute or more", () => { expect(decodeRange({ ...valid, projects: ["acme-prod"] })._tag).toBe("Success") + // Exactly 11,000 points. + expect( + decodeRange({ ...valid, startTime: "2026-10-15 00:00:00", endTime: "2026-10-22 15:20:00" })._tag, + ).toBe("Success") }) it.each([ @@ -22,6 +26,11 @@ describe("GcpPromQlRangeRequest", () => { ["a fractional step", { stepSeconds: 90.5 }], ["a step over a day", { stepSeconds: 86_401 }], ["more than 11,000 points a series", { startTime: "2026-09-01 00:00:00" }], + // 11,000 steps as given, 11,001 once start and end are aligned to the step. + [ + "more than 11,000 points on the aligned range", + { startTime: "2026-10-15 00:00:30", endTime: "2026-10-22 15:20:30" }, + ], ["an inverted window", { startTime: valid.endTime, endTime: valid.startTime }], ["an impossible date", { endTime: "2026-13-45 99:99:99" }], ["a project id that is not one", { projects: ["../../organizations/1"] }], diff --git a/packages/domain/src/http/gcp-promql.ts b/packages/domain/src/http/gcp-promql.ts index 2f4e36a29c..19275d9db0 100644 --- a/packages/domain/src/http/gcp-promql.ts +++ b/packages/domain/src/http/gcp-promql.ts @@ -21,6 +21,20 @@ export const GCP_METRIC_SEARCH_MAX_LIMIT = 200 /** `TinybirdDateTime` is UTC without a zone marker. */ export const gcpPromQlTimeMs = (value: string): number => Date.parse(`${value.replace(" ", "T")}Z`) +/** The range Google is sent, in epoch seconds: start and end aligned to the step. */ +export const gcpPromQlRange = (request: { + readonly startTime: string + readonly endTime: string + readonly stepSeconds: number +}) => { + const step = request.stepSeconds + return { + start: Math.floor(gcpPromQlTimeMs(request.startTime) / 1000 / step) * step, + end: Math.ceil(gcpPromQlTimeMs(request.endTime) / 1000 / step) * step, + step, + } +} + const PromQlQuery = Schema.String.check( Schema.isTrimmed(), Schema.isMinLength(1), @@ -56,11 +70,13 @@ export class GcpPromQlRangeRequest extends Schema.Class(" readonly endTime: string readonly stepSeconds: number }) => { - const extentMs = gcpPromQlTimeMs(request.endTime) - gcpPromQlTimeMs(request.startTime) - if (extentMs <= 0) return "endTime must be after startTime" - // Google's limit per series, refused here as a request error. + if (gcpPromQlTimeMs(request.endTime) <= gcpPromQlTimeMs(request.startTime)) { + return "endTime must be after startTime" + } + // Google's limit per series, on the range Google is sent, refused here as a request error. + const { start, end, step } = gcpPromQlRange(request) return ( - extentMs / 1000 / request.stepSeconds <= GCP_PROMQL_MAX_POINTS || + (end - start) / step <= GCP_PROMQL_MAX_POINTS || `at most ${GCP_PROMQL_MAX_POINTS} points per series: widen stepSeconds` ) }, @@ -78,9 +94,15 @@ export class GcpPromQlInstantRequest extends Schema.Class @@ -96,7 +118,10 @@ export type GcpPromQlFailure = Schema.Schema.Type export class GcpPromQlResult extends Schema.Class("GcpPromQlResult")({ resultType: Schema.Literals(["matrix", "vector"]), series: Schema.Array(GcpPromQlSeries), - /** The projects that answered. */ + /** + * The projects that answered. A project another connected project's metrics scope holds is + * read through that one, once. + */ projects: Schema.Array(GcpProjectId), /** What could not be read. The rest is still returned. */ failures: Schema.Array(GcpPromQlFailure), @@ -158,10 +183,13 @@ export class GcpPromQlTooManyProjectsError extends HttpTaggedError()( "@maple/http/errors/GcpPromQlTooManySeriesError", - { message: Schema.String, series: Schema.Number }, + { message: Schema.String }, { status: 400, code: "gcp_promql_too_many_series", From 0c368708a65b5612a3944f0924dad444941ae8d8 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 11:58:08 +0200 Subject: [PATCH 6/8] fix(gcp): cap only live-query answers, leave the poller's reads uncapped --- .../services/integrations/GcpPromQlService.ts | 7 +- .../src/services/integrations/gcp/api.test.ts | 33 +++++++++ .../src/services/integrations/gcp/api.ts | 69 +++++++++++++++---- 3 files changed, 89 insertions(+), 20 deletions(-) create mode 100644 packages/backend/src/services/integrations/gcp/api.test.ts diff --git a/packages/backend/src/services/integrations/GcpPromQlService.ts b/packages/backend/src/services/integrations/GcpPromQlService.ts index 219b0e8c8f..eaabf193f1 100644 --- a/packages/backend/src/services/integrations/GcpPromQlService.ts +++ b/packages/backend/src/services/integrations/GcpPromQlService.ts @@ -8,7 +8,6 @@ import * as PG from "@maple-dev/effect-orm/postgres" import { EdgeCacheService } from "@maple/cache" import { GcpConnectors, type GcpConnectorRow } from "@maple/db/tables" -import { GCP_PROJECT_ASSET_TYPE } from "@maple/domain/gcp-metrics" import { GCP_PROMQL_MAX_PROJECTS, GcpMetricDescriptor, @@ -48,7 +47,7 @@ import { impersonateReader, listPromQlMetricNames, promQlQuery, - searchResourcesPage, + searchProjectsPage, } from "./gcp/api" import { describeGcpApiError } from "./gcp/explain" import { mapGcpResource } from "./gcp/mapping" @@ -213,9 +212,7 @@ export class GcpPromQlService extends Context.Service = [] let pageToken: string | undefined for (let page = 0; page < MAX_PROJECT_PAGES; page++) { - const body = yield* searchResourcesPage(httpClient, reader, scope, pageToken, [ - GCP_PROJECT_ASSET_TYPE, - ]) + const body = yield* searchProjectsPage(httpClient, reader, scope, pageToken) for (const result of body.results ?? []) { const projectId = mapGcpResource(result)?.projectId // A project pending deletion answers nothing but errors. diff --git a/packages/backend/src/services/integrations/gcp/api.test.ts b/packages/backend/src/services/integrations/gcp/api.test.ts new file mode 100644 index 0000000000..bc25d87e2f --- /dev/null +++ b/packages/backend/src/services/integrations/gcp/api.test.ts @@ -0,0 +1,33 @@ +import { assert, describe, it } from "@effect/vitest" +import { GCP_METRIC_GROUPS } from "@maple/domain/gcp-metrics" +import { GcpProjectId } from "@maple/domain/primitives" +import { Effect, Redacted, Schema } from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import { listTimeSeriesPage, MAX_BODY_BYTES } from "./api" + +const reader = { + accessToken: Redacted.make("ya29.reader"), + quotaProject: Schema.decodeSync(GcpProjectId)("acme-prod"), +} +const group = GCP_METRIC_GROUPS[0]! + +describe("Google calls the poller makes", () => { + it.effect("read a page past the live queries' size cap in full", () => { + const label = "x".repeat(MAX_BODY_BYTES + 1) + const stub = (async () => + new Response( + JSON.stringify({ timeSeries: [{ metric: { labels: { label } }, resource: { labels: {} } }] }), + { headers: { "content-type": "application/json" } }, + )) as typeof fetch + return Effect.gen(function* () { + const httpClient = yield* HttpClient.HttpClient + const page = yield* listTimeSeriesPage( + httpClient, + reader, + { scope: "projects/acme-prod", group, metric: group.metrics[0]!, startMs: 0, endMs: 60_000 }, + undefined, + ) + assert.strictEqual(page.timeSeries?.[0]?.metric.labels?.label, label) + }).pipe(Effect.provide(FetchHttpClient.layer), Effect.provideService(FetchHttpClient.Fetch, stub)) + }) +}) diff --git a/packages/backend/src/services/integrations/gcp/api.ts b/packages/backend/src/services/integrations/gcp/api.ts index 5ffc5e52d8..34e0834b61 100644 --- a/packages/backend/src/services/integrations/gcp/api.ts +++ b/packages/backend/src/services/integrations/gcp/api.ts @@ -12,6 +12,7 @@ import { gcpConnectorResourceNames } from "@maple/domain/gcp" import { GCP_ASSET_TYPES, + GCP_PROJECT_ASSET_TYPE, gcpMetricAligner, type GcpMetric, type GcpMetricGroup, @@ -118,25 +119,37 @@ const errorReason = (body: unknown, kind: GcpApiError["kind"]): string | undefin .flatMap((detail) => Option.toArray(decodeErrorInfo(detail)))[0]?.reason ?? (kind === "denied" ? reasonFromMessage(body) : undefined) -/** The largest answer read. A poller page is about 1 MB; a bigger PromQL answer is refused. */ +/** + * The largest answer a live query reads: four in flight stay far below the Worker's 128 MB. + * The poller's calls are not capped. + */ export const MAX_BODY_BYTES = 8 * 1024 * 1024 const parseJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)) -/** - * One JSON call. `api` names the Google service in the error. `refuse` turns a failed answer, - * and `tooLarge` one past `MAX_BODY_BYTES`, into the caller's own error instead. - */ +interface CallOptions { + /** Turns a failed answer into the caller's own error. */ + readonly refuse?: (status: number, body: unknown) => E | undefined + /** Reads at most `MAX_BODY_BYTES`; `tooLarge` is the caller's error past it. */ + readonly capped?: { readonly tooLarge?: () => E2 } +} + +/** One JSON call. `api` names the Google service in the error. */ const call = ( httpClient: HttpClient.HttpClient, api: GcpApiError["api"], request: HttpClientRequest.HttpClientRequest, decode: (body: unknown) => Effect.Effect, - refuse?: (status: number, body: unknown) => E | undefined, - tooLarge?: () => E2, + options: CallOptions = {}, ): Effect.Effect => { const failed = (what: string) => new GcpApiError({ message: `${api} ${what}`, api, kind: "upstream" }) return Effect.gen(function* () { const response = yield* httpClient.execute(request) + if (options.capped === undefined) { + return { + status: response.status, + body: yield* response.json.pipe(Effect.orElseSucceed(() => null)), + } + } // Read as it arrives, never trusting Content-Length: past the cap the rest is left unread. const chunks: Array = [] let size = 0 @@ -161,11 +174,12 @@ const call = ( Effect.flatMap(({ status, body }) => Effect.gen(function* () { if (body === undefined) { + const tooLarge = options.capped?.tooLarge if (tooLarge !== undefined) return yield* Effect.fail(tooLarge()) return yield* failed("answered with more than 8 MB") } if (status >= 300) { - const refused = refuse?.(status, body) + const refused = options.refuse?.(status, body) if (refused !== undefined) return yield* Effect.fail(refused) const kind = errorKind(status) const reason = errorReason(body, kind) @@ -499,11 +513,15 @@ export const promQlQuery = ( HttpClientRequest.bodyUrlParams({ ...params, timeout: PROMQL_TIMEOUT }), ), decodePromQlResult, - refuseQuery(projectId), - () => - new GcpPromQlTooManySeriesError({ - message: `Google's answer for project ${projectId} is larger than 8 MB: too many series or points. Aggregate with sum by (…) or avg by (…), narrow the selector, or widen the step.`, - }), + { + refuse: refuseQuery(projectId), + capped: { + tooLarge: () => + new GcpPromQlTooManySeriesError({ + message: `Google's answer for project ${projectId} is larger than 8 MB: too many series or points. Aggregate with sum by (…) or avg by (…), narrow the selector, or widen the step.`, + }), + }, + }, ).pipe( Effect.flatMap(({ data }) => Effect.gen(function* () { @@ -566,6 +584,7 @@ export const getMetricsScope = ( `https://monitoring.googleapis.com/v1/locations/global/metricsScopes/${projectId}`, ).pipe(asReader(reader)), decodeMetricsScope, + { capped: {} }, ).pipe( Effect.map((scope) => ({ number: lastSegment(scope.name), @@ -590,6 +609,7 @@ export const listPromQlMetricNames = ( "Cloud Monitoring", HttpClientRequest.get(promQlUrl(projectId, "label/__name__/values")).pipe(asReader(reader)), decodeNames, + { capped: {} }, ).pipe( Effect.map(({ data }) => { const legacy = new Set( @@ -641,6 +661,7 @@ export const getMetricDescriptor = ( `https://monitoring.googleapis.com/v3/projects/${projectId}/metricDescriptors`, ).pipe(asReader(reader), HttpClientRequest.setUrlParams({ filter: `metric.type = "${metricType}"` })), decodeMetricDescriptors, + { capped: {} }, ).pipe(Effect.map((body) => body.metricDescriptors?.find((descriptor) => descriptor.type === metricType))) // Cloud Asset Inventory @@ -667,12 +688,13 @@ const decodeResourcePage = Schema.decodeUnknownEffect( }), ) -export const searchResourcesPage = ( +const resourcePage = ( httpClient: HttpClient.HttpClient, reader: GcpReader, scope: string, pageToken: string | undefined, - assetTypes: ReadonlyArray = GCP_ASSET_TYPES, + assetTypes: ReadonlyArray, + options: CallOptions = {}, ) => call( httpClient, @@ -687,4 +709,21 @@ export const searchResourcesPage = ( }), ), decodeResourcePage, + options, ) + +/** One page of the resource inventory sync. */ +export const searchResourcesPage = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + scope: string, + pageToken: string | undefined, +) => resourcePage(httpClient, reader, scope, pageToken, GCP_ASSET_TYPES) + +/** One page of the projects in a folder or organization, for a live query. */ +export const searchProjectsPage = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + scope: string, + pageToken: string | undefined, +) => resourcePage(httpClient, reader, scope, pageToken, [GCP_PROJECT_ASSET_TYPE], { capped: {} }) From 901ee1e0cb9cd68bc0b9debafad2c673c74c8e18 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 23:20:50 +0200 Subject: [PATCH 7/8] fix(ci): raise the web bundle budget for #1475 --- apps/web/perf/check-bundle-budget.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/web/perf/check-bundle-budget.ts b/apps/web/perf/check-bundle-budget.ts index 7bafbe4eaf..34733bc627 100644 --- a/apps/web/perf/check-bundle-budget.ts +++ b/apps/web/perf/check-bundle-budget.ts @@ -107,7 +107,8 @@ const gzipBytes = chunks.reduce((total, chunk) => total + chunk.gzipBytes, 0) // schemas on the contract, the connector status rules the sidebar and the // integrations hub read, and the Infrastructure nav entry with its icon. The // /infra/gcp pages stay in their route chunks. -const maxGzipBytes = 700.5 * 1024 +// 701.5 KB from the PromQL query contract (#1475, 2026-10-15), measured in CI at 701.3. +const maxGzipBytes = 701.5 * 1024 const budgetLabel = `${(maxGzipBytes / 1024).toFixed(1)} KB` // Anything lazy-only: chat, replay, and every dev-only lab surface. The From 92e2d0ff94c9a320b780180e4abf51a387ad2c24 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 23:32:40 +0200 Subject: [PATCH 8/8] fix(ci): resolve the bundle budget merge --- apps/web/perf/check-bundle-budget.ts | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/apps/web/perf/check-bundle-budget.ts b/apps/web/perf/check-bundle-budget.ts index c4b7708a84..d10705de1e 100644 --- a/apps/web/perf/check-bundle-budget.ts +++ b/apps/web/perf/check-bundle-budget.ts @@ -96,30 +96,20 @@ const gzipBytes = chunks.reduce((total, chunk) => total + chunk.gzipBytes, 0) // carries, the pill in the persistent top bar with its live clock, and the // hook's query and seen-flag atoms. The popover's panel and pointer are small // and render only while open, so nothing there is worth a lazy chunk. -// 699.5 KB from the Google Cloud connectors (#1321, 2026-10-15): ~2.5 KB of +// 698.5 KB from the Google Cloud connectors (#1321, 2026-10-15): ~2.5 KB of // startup measured in CI against main's 694.6 — the connector group's endpoints -// and schemas on the v2 contract every page's client carries. -<<<<<<< HEAD -// 698.5 KB from the Google Cloud integration card (#1334, 2026-10-15): ~0.9 KB +// and schemas on the v2 contract every page's client carries. Each Google Cloud +// step below keeps ~1 KB of headroom, since CI measures the PR merged with main. +// 699.5 KB from the Google Cloud integration card (#1334, 2026-10-15): ~0.9 KB // of startup measured in CI against #1333's 697.3 from the card and its // contract additions. -<<<<<<< HEAD -// 702.5 KB from the Infrastructure → Google Cloud page (#1336, 2026-10-15): +// 701.5 KB from the Infrastructure → Google Cloud page (#1336, 2026-10-15): // ~2 KB of startup measured in CI against #1335's 698.3 — the inventory // schemas on the contract, the connector status rules the sidebar and the // integrations hub read, and the Infrastructure nav entry with its icon. The // /infra/gcp pages stay in their route chunks. -<<<<<<< HEAD -// 701.5 KB from the PromQL query contract (#1475, 2026-10-15), measured in CI at 701.3. +// 702.5 KB from the PromQL query contract (#1475, 2026-10-15), measured in CI at 701.3. const maxGzipBytes = 702.5 * 1024 -======= -const maxGzipBytes = 702.5 * 1024 -======= -======= ->>>>>>> origin/feat/gcp-metrics-poller -const maxGzipBytes = 699.5 * 1024 ->>>>>>> origin/feat/gcp-dashboard-template ->>>>>>> origin/feat/gcp-infra-page const budgetLabel = `${(maxGzipBytes / 1024).toFixed(1)} KB` // Anything lazy-only: chat, replay, and every dev-only lab surface. The