From 6e69eccb53f560e746d4ab1158e0dbf851bb808c Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 21:43:54 +0200 Subject: [PATCH 01/51] feat(infra): Google Cloud service tab queries --- packages/domain/package.json | 1 + packages/domain/src/gcp-infra.ts | 88 +++++++++++++ .../src/benchmark/index.ts | 15 +++ .../src/gcp/gcp-infra.test.ts | 72 +++++++++++ .../src/gcp/gcp-infra.ts | 117 ++++++++++++++++++ .../src/gcp/index.ts | 8 ++ .../query-engine-integrations/src/index.ts | 1 + 7 files changed, 302 insertions(+) create mode 100644 packages/domain/src/gcp-infra.ts create mode 100644 packages/query-engine-integrations/src/gcp/gcp-infra.test.ts create mode 100644 packages/query-engine-integrations/src/gcp/gcp-infra.ts create mode 100644 packages/query-engine-integrations/src/gcp/index.ts diff --git a/packages/domain/package.json b/packages/domain/package.json index 3944657fc7..cf611ca9be 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -12,6 +12,7 @@ "./billing-tax-ids": "./src/billing-tax-ids.ts", "./format": "./src/format.ts", "./gcp": "./src/gcp.ts", + "./gcp-infra": "./src/gcp-infra.ts", "./gcp-metrics": "./src/gcp-metrics.ts", "./gen-ai": "./src/gen-ai.ts", "./glob": "./src/glob.ts", diff --git a/packages/domain/src/gcp-infra.ts b/packages/domain/src/gcp-infra.ts new file mode 100644 index 0000000000..b599247dc7 --- /dev/null +++ b/packages/domain/src/gcp-infra.ts @@ -0,0 +1,88 @@ +/** + * The Infrastructure -> Google Cloud page: one tab per curated service, a table with one row per + * workload. A tab reads one group of `GCP_METRIC_GROUPS`, and a row is identified by resource + * attributes the metrics poller stores. + */ +import { GCP_METRIC_GROUPS, type GcpMetric } from "./gcp-metrics" + +export const GCP_INFRA_SERVICE_IDS = [ + "cloudRun", + "cloudFunctions", + "gke", + "computeEngine", + "cloudSql", + "pubsub", + "loadBalancing", +] as const +export type GcpInfraServiceId = (typeof GCP_INFRA_SERVICE_IDS)[number] + +export interface GcpInfraService { + readonly title: string + /** The metric group the tab reads. */ + readonly resourceType: string + /** What identifies a row, as column label and resource attribute. The first names the row. */ + readonly identity: ReadonlyArray +} + +const PROJECT = ["Project", "cloud.account.id"] as const +const REGION = ["Region", "cloud.region"] as const +const resourceLabel = (name: string) => `gcp.resource.labels.${name}` + +export const GCP_INFRA_SERVICES: Record = { + // Includes Cloud Functions (2nd gen), which run on Cloud Run. + cloudRun: { + title: "Cloud Run", + resourceType: "cloud_run_revision", + identity: [["Service", "service.name"], PROJECT, REGION], + }, + // 1st gen. + cloudFunctions: { + title: "Cloud Functions", + resourceType: "cloud_function", + identity: [["Function", "service.name"], PROJECT, REGION], + }, + // One row per container name: Cloud Monitoring has already folded the pods together. + gke: { + title: "GKE", + resourceType: "k8s_container", + identity: [ + ["Container", "k8s.container.name"], + ["Namespace", "k8s.namespace.name"], + ["Cluster", "k8s.cluster.name"], + PROJECT, + ["Location", resourceLabel("location")], + ], + }, + computeEngine: { + title: "Compute Engine", + resourceType: "gce_instance", + identity: [["Instance", "host.name"], PROJECT, ["Zone", "cloud.availability_zone"]], + }, + cloudSql: { + title: "Cloud SQL", + resourceType: "cloudsql_database", + identity: [["Instance", resourceLabel("database_id")], PROJECT, REGION], + }, + pubsub: { + title: "Pub/Sub", + resourceType: "pubsub_subscription", + identity: [["Subscription", resourceLabel("subscription_id")], PROJECT], + }, + loadBalancing: { + title: "Load Balancing", + resourceType: "https_lb_rule", + identity: [ + ["URL map", resourceLabel("url_map_name")], + ["Backend", resourceLabel("backend_target_name")], + PROJECT, + ], + }, +} satisfies Record + +/** The curated metrics behind a tab. */ +export const gcpInfraMetrics = (service: GcpInfraServiceId): ReadonlyArray => + GCP_METRIC_GROUPS.find((group) => group.resourceType === GCP_INFRA_SERVICES[service].resourceType) + ?.metrics ?? [] + +/** Cap on the rows of one tab's query: one row per workload, metric and label value. */ +export const GCP_INFRA_ROW_LIMIT = 10_000 diff --git a/packages/query-engine-integrations/src/benchmark/index.ts b/packages/query-engine-integrations/src/benchmark/index.ts index d97df8f6f7..467ff08ef4 100644 --- a/packages/query-engine-integrations/src/benchmark/index.ts +++ b/packages/query-engine-integrations/src/benchmark/index.ts @@ -10,6 +10,7 @@ import { Effect } from "effect" import { compileUnionUnsafe, compileUnsafe, type CompiledQuery } from "@maple/query-engine/ch" +import { GCP_INFRA_SERVICE_IDS } from "@maple/domain/gcp-infra" import { MAPLE_AI_TRACE_SESSION_PREFIX } from "@maple/domain/gen-ai" import * as CH from "../index" import { BenchmarkError, type CatalogEntry } from "@maple/query-engine/benchmark" @@ -758,6 +759,20 @@ export const integrationFixtures: ReadonlyArray = [ label: "default", compile: () => compileUnsafe(CH.railwayServiceTimeseriesSQL(), railwayServiceBucketed), }, + { + module: "gcp-infra", + name: "gcpInfraPresenceSQL", + label: "default", + compile: () => compileUnsafe(CH.gcpInfraPresenceSQL(), window), + }, + ...GCP_INFRA_SERVICE_IDS.map( + (service): IntegrationFixture => ({ + module: "gcp-infra", + name: "gcpInfraMetricsSQL", + label: service, + compile: () => compileUnsafe(CH.gcpInfraMetricsSQL(service), window), + }), + ), { module: "planetscale-map", name: "planetscaleBranchGaugesSQL", diff --git a/packages/query-engine-integrations/src/gcp/gcp-infra.test.ts b/packages/query-engine-integrations/src/gcp/gcp-infra.test.ts new file mode 100644 index 0000000000..3b2f97b610 --- /dev/null +++ b/packages/query-engine-integrations/src/gcp/gcp-infra.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" +import { OrgId } from "@maple/domain" +import { GCP_INFRA_SERVICE_IDS, gcpInfraMetrics } from "@maple/domain/gcp-infra" +import { gcpInfraMetricsSQL, gcpInfraPresenceSQL } from "./gcp-infra" + +const params = { + orgId: OrgId.make("org_1"), + startTime: "2026-07-02 00:00:00.000", + endTime: "2026-07-03 00:00:00.000", +} + +describe("gcpInfraMetricsSQL", () => { + it("reads counters and gauges of one service in one grouped query", () => { + const { sql, tenantScope } = compileUnsafe(gcpInfraMetricsSQL("cloudRun"), params) + expect(tenantScope).toBe("single-tenant") + expect(sql).toContain( + "FROM metrics_sum\n WHERE metrics_sum.OrgId = 'org_1'\n AND metrics_sum.MetricName IN ('gcp.run.request_count', 'gcp.run.container.billable_instance_time')", + ) + expect(sql).toContain("FROM metrics_gauge\n WHERE metrics_gauge.OrgId = 'org_1'") + expect(sql).toContain("'gcp.run.request_latencies', 'gcp.run.container.instance_count'") + expect(sql.match(/TimeUnix >= '2026-07-02 00:00:00.000'/g)).toHaveLength(2) + expect(sql.match(/TimeUnix <= '2026-07-03 00:00:00.000'/g)).toHaveLength(2) + expect(sql).toContain( + "[metrics_sum.ResourceAttributes['service.name'], metrics_sum.ResourceAttributes['cloud.account.id'], metrics_sum.ResourceAttributes['cloud.region']] AS keys", + ) + // The quantile and the metric's own label share one column. + expect(sql).toContain( + "arrayStringConcat([metrics_gauge.Attributes['quantile'], metrics_gauge.Attributes['response_code_class'], metrics_gauge.Attributes['state']], '') AS label", + ) + expect(sql).toContain("GROUP BY keys, metric, label") + expect(sql).toContain("LIMIT 10000") + expect(sql).toContain("FORMAT JSON") + }) + + it("keeps every service to its own metrics and never leaves a branch without one", () => { + for (const service of GCP_INFRA_SERVICE_IDS) { + const { sql } = compileUnsafe(gcpInfraMetricsSQL(service), params) + expect(sql, service).not.toContain("IN ()") + // Metric names only: `gcp.resource.labels.*` are identity attributes. + const names = [...sql.matchAll(/'(gcp\.(?!resource\.)[a-z_.]+)'/g)].map((match) => match[1]) + expect(names.sort(), service).toEqual( + gcpInfraMetrics(service) + .map((metric) => metric.name) + .sort(), + ) + } + }) + + it("escapes single quotes in orgId", () => { + const { sql } = compileUnsafe(gcpInfraMetricsSQL("gke"), { ...params, orgId: OrgId.make("org'evil") }) + expect(sql).toContain("OrgId = 'org\\'evil'") + expect(sql).not.toContain("OrgId = 'org'evil'") + }) +}) + +describe("gcpInfraPresenceSQL", () => { + it("probes the hourly catalog for the page's metrics, flooring the start to the hour", () => { + const { sql, tenantScope } = compileUnsafe(gcpInfraPresenceSQL(), params) + expect(tenantScope).toBe("single-tenant") + expect(sql).toContain("FROM metric_catalog") + expect(sql).toContain("metric_catalog.OrgId = 'org_1'") + for (const service of GCP_INFRA_SERVICE_IDS) { + expect(sql, service).toContain(`'${gcpInfraMetrics(service)[0]?.name}'`) + } + expect(sql).toContain( + "Hour >= toStartOfInterval(toDateTime('2026-07-02 00:00:00.000'), INTERVAL 3600 SECOND)", + ) + expect(sql).toContain("Hour <= '2026-07-03 00:00:00'") + expect(sql).toContain("GROUP BY metric") + }) +}) diff --git a/packages/query-engine-integrations/src/gcp/gcp-infra.ts b/packages/query-engine-integrations/src/gcp/gcp-infra.ts new file mode 100644 index 0000000000..9295ce44dc --- /dev/null +++ b/packages/query-engine-integrations/src/gcp/gcp-infra.ts @@ -0,0 +1,117 @@ +// Infrastructure -> Google Cloud page (/infra/gcp) +// +// Reads the `gcp.*` metrics the Google Cloud poller stores, one point per minute and series. +// Counters are delta sums in `metrics_sum`; gauges and percentiles (one row per `quantile`) are in +// `metrics_gauge`. Nothing else writes these metric names, so they scope a query on their own. + +import * as CH from "@maple-dev/effect-orm/expr" +import { from, fromUnion, param, unionAll, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" +import { + GCP_INFRA_ROW_LIMIT, + GCP_INFRA_SERVICE_IDS, + GCP_INFRA_SERVICES, + gcpInfraMetrics, + type GcpInfraServiceId, +} from "@maple/domain/gcp-infra" +import { + MetricCatalog, + MetricsGauge, + MetricsSum, + orgIdParam, + utcSecondsParam, +} from "@maple/query-engine/ch/tables" + +/** Both metrics tables declare the columns read here alike. */ +type MetricsAccessor = + | ColumnAccessor + | ColumnAccessor + +export interface GcpInfraMetricsOutput { + /** The workload's identity values, in the order of the service's `identity`. */ + readonly keys: ReadonlyArray + readonly metric: string + /** The metric's label value or quantile; empty when it has neither. */ + readonly label: string + /** Sum of the points in the window. */ + readonly total: number + /** Number of points in the window. */ + readonly samples: number +} + +/** + * Every curated metric of one service: one row per workload, metric and label value. A counter + * reads as `total`, a gauge or percentile as the mean `total / samples`. + */ +export function gcpInfraMetricsSQL(service: GcpInfraServiceId) { + const metrics = gcpInfraMetrics(service) + // A metric keeps at most one label, so joining the candidates yields the one that is set. + const labels = ["quantile", ...new Set(metrics.flatMap((metric) => metric.labels))] + const point = ($: MetricsAccessor) => ({ + keys: CH.arrayOf( + ...GCP_INFRA_SERVICES[service].identity.map(([, attribute]) => + $.ResourceAttributes.get(attribute), + ), + ), + metric: $.MetricName, + label: CH.arrayStringConcat( + labels.map((label) => $.Attributes.get(label)), + "", + ), + value: $.Value, + }) + const inWindow = ($: MetricsAccessor, sums: boolean) => [ + $.OrgId.eq(orgIdParam), + CH.inList( + $.MetricName, + metrics.filter((metric) => (metric.kind === "sum") === sums).map((metric) => metric.name), + ), + $.TimeUnix.gte(param.dateTime("startTime")), + $.TimeUnix.lte(param.dateTime("endTime")), + ] + return fromUnion( + unionAll( + from(MetricsSum) + .select(point) + .where(($) => inWindow($, true)), + from(MetricsGauge) + .select(point) + .where(($) => inWindow($, false)), + ), + "points", + ) + .select(($) => ({ + keys: $.keys, + metric: $.metric, + label: $.label, + total: CH.sum($.value), + samples: CH.count(), + })) + .groupBy("keys", "metric", "label") + .orderBy(["keys", "asc"], ["metric", "asc"], ["label", "asc"]) + .limit(GCP_INFRA_ROW_LIMIT) + .format("JSON") +} + +export interface GcpInfraPresenceOutput { + readonly metric: string +} + +/** Which of the page's metrics reported in the window, from the hourly `metric_catalog` rollup. */ +export function gcpInfraPresenceSQL() { + return from(MetricCatalog) + .select(($) => ({ metric: $.MetricName })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + CH.inList( + $.MetricName, + GCP_INFRA_SERVICE_IDS.flatMap((service) => + gcpInfraMetrics(service).map((metric) => metric.name), + ), + ), + // `Hour` is hour-truncated: floor the start so a mid-hour range keeps its first bucket. + $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(param.dateTime("startTime")), 3600)), + $.Hour.lte(utcSecondsParam("endTime")), + ]) + .groupBy("metric") + .format("JSON") +} diff --git a/packages/query-engine-integrations/src/gcp/index.ts b/packages/query-engine-integrations/src/gcp/index.ts new file mode 100644 index 0000000000..30a5987a30 --- /dev/null +++ b/packages/query-engine-integrations/src/gcp/index.ts @@ -0,0 +1,8 @@ +// Google Cloud integration queries: the /infra/gcp service tabs. + +export { + gcpInfraMetricsSQL, + gcpInfraPresenceSQL, + type GcpInfraMetricsOutput, + type GcpInfraPresenceOutput, +} from "./gcp-infra" diff --git a/packages/query-engine-integrations/src/index.ts b/packages/query-engine-integrations/src/index.ts index 64f1304c40..d2bd3bcb28 100644 --- a/packages/query-engine-integrations/src/index.ts +++ b/packages/query-engine-integrations/src/index.ts @@ -14,6 +14,7 @@ export * from "./ai/index" export * from "./cloudflare/index" +export * from "./gcp/index" export * from "./planetscale/index" export * from "./product/index" export * from "./railway/index" From 68ba630fdd68c2332f46b2b1d1039da33db620c7 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 21:52:52 +0200 Subject: [PATCH 02/51] feat(infra): Google Cloud infra endpoints, inventory read path and row helpers --- .../src/routes/internal/integrations.http.ts | 9 + .../src/routes/internal/query-engine.http.ts | 22 ++ apps/api/src/routes/queries.ts | 29 ++ apps/web/src/api/warehouse/gcp-infra.ts | 53 ++++ .../web/src/components/infra/gcp/tabs.test.ts | 242 +++++++++++++++ apps/web/src/components/infra/gcp/tabs.ts | 290 ++++++++++++++++++ .../services/atoms/warehouse-query-atoms.ts | 9 + .../integrations/GcpConnectorService.test.ts | 81 +++++ .../integrations/GcpConnectorService.ts | 92 ++++++ packages/domain/src/http/integrations.ts | 41 +++ packages/domain/src/http/query-engine.ts | 53 ++++ .../src/gcp/gcp-infra.ts | 22 +- .../src/gcp/index.ts | 7 +- 13 files changed, 926 insertions(+), 24 deletions(-) create mode 100644 apps/web/src/api/warehouse/gcp-infra.ts create mode 100644 apps/web/src/components/infra/gcp/tabs.test.ts create mode 100644 apps/web/src/components/infra/gcp/tabs.ts diff --git a/apps/api/src/routes/internal/integrations.http.ts b/apps/api/src/routes/internal/integrations.http.ts index d7314c7e59..6c64713cb4 100644 --- a/apps/api/src/routes/internal/integrations.http.ts +++ b/apps/api/src/routes/internal/integrations.http.ts @@ -10,6 +10,7 @@ import { CloudflareUsageResponse, CurrentTenant, ExternalUserId, + GcpResourcesResponse, GithubDeleteRepositoryResponse, GithubDisconnectResponse, GithubIntegrationStatus, @@ -58,6 +59,7 @@ import { topTrafficQuery, type TopTrafficGroupDefinition, } from "@maple/backend/services/integrations/cloudflare-analytics/queries" +import { GcpConnectorService } from "@maple/backend/services/integrations/GcpConnectorService" import { RailwayMetricsService } from "@maple/backend/services/integrations/RailwayMetricsService" import { GithubConnectService } from "@maple/backend/services/integrations/vcs/vendor/github/GithubConnectService" import { VcsCommitService } from "@maple/backend/services/integrations/vcs/VcsCommitService" @@ -122,6 +124,7 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte const cloudflare = yield* CloudflareOAuthService const cloudflareAnalytics = yield* CloudflareAnalyticsService const railway = yield* RailwayMetricsService + const gcp = yield* GcpConnectorService const database = yield* Database const edgeCache = yield* EdgeCacheService const env = yield* Env @@ -504,6 +507,12 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte return new RailwayDisconnectResponse(yield* railway.disconnect(tenant.orgId)) }), ) + .handle("gcpResources", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + return new GcpResourcesResponse(yield* gcp.resources(tenant.orgId, query)) + }), + ) .handle("githubStatus", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index beaa787be3..33840ba4fb 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -18,6 +18,8 @@ import { ServiceApdexResponse, PlanetScaleInfraTimeseriesResponse, RailwayInfraRowsResponse, + GcpInfraMetricsResponse, + GcpInfraPresenceResponse, ServiceCloudflareStatsResponse, ServicePlanetScaleStatsResponse, CloudflareInfraZonesResponse, @@ -94,6 +96,7 @@ import { TraceId, SpanId, } from "@maple/domain/http" +import { GCP_INFRA_SERVICE_IDS, gcpInfraMetrics } from "@maple/domain/gcp-infra" import { SESSION_LIVE_WINDOW_SECONDS } from "@maple/domain/query-engine" import { isAiContentFormat } from "@maple/domain/ai-traffic" import { Cause, Clock, DateTime, Effect, Option, Schema } from "effect" @@ -797,6 +800,25 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) + .handle("gcpInfraPresence", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpInfraPresence, tenant, payload) + const reported = new Set(rows.map((row) => row.metric)) + return new GcpInfraPresenceResponse({ + services: GCP_INFRA_SERVICE_IDS.filter((service) => + gcpInfraMetrics(service).some((metric) => reported.has(metric.name)), + ), + }) + }), + ) + .handle("gcpInfraMetrics", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpInfraMetrics, tenant, payload) + return new GcpInfraMetricsResponse({ data: rows }) + }), + ) .handle("cloudflareInfraZones", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/api/src/routes/queries.ts b/apps/api/src/routes/queries.ts index a844c89eb7..2d5b0b4184 100644 --- a/apps/api/src/routes/queries.ts +++ b/apps/api/src/routes/queries.ts @@ -15,6 +15,8 @@ import type { CloudflareInfraZoneTimeseriesRequest, CloudflareInfraZonesRequest, ContainerInfraTimeseriesRequest, + GcpInfraMetricsRequest, + GcpInfraPresenceRequest, GetLogRequest, NodeInfraTimeseriesRequest, PlanetScaleInfraTimeseriesRequest, @@ -287,6 +289,31 @@ const railwayInfraServiceTimeseries = defineQuery({ }), }) +// A small read of the hourly catalog that decides which tabs /infra/gcp shows. +const gcpInfraPresence = defineQuery({ + id: "gcpInfraPresence", + profile: "discovery", + cache: 60, + compile: (payload: GcpInfraPresenceRequest, orgId: OrgId) => + CH.compile(Integrations.gcpInfraPresenceSQL(), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), +}) + +const gcpInfraMetrics = defineQuery({ + id: "gcpInfraMetrics", + profile: "aggregation", + cache: 15, + compile: (payload: GcpInfraMetricsRequest, orgId: OrgId) => + CH.compile(Integrations.gcpInfraMetricsSQL(payload.service), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), +}) + const zoneDetailParams = (payload: CloudflareInfraZoneDetailRequest, orgId: OrgId) => ({ orgId, serviceName: payload.serviceName, @@ -713,6 +740,8 @@ export const Queries = { planetscaleInfraTimeseries, railwayInfraServices, railwayInfraServiceTimeseries, + gcpInfraPresence, + gcpInfraMetrics, // ZoneDetail / PlanetScaleStats sub-queries, declared above. cloudflareInfraZoneDetailStatus, diff --git a/apps/web/src/api/warehouse/gcp-infra.ts b/apps/web/src/api/warehouse/gcp-infra.ts new file mode 100644 index 0000000000..9a1f3d25fc --- /dev/null +++ b/apps/web/src/api/warehouse/gcp-infra.ts @@ -0,0 +1,53 @@ +import { Effect, Schema } from "effect" +import { GCP_INFRA_SERVICE_IDS } from "@maple/domain/gcp-infra" +import { GcpInfraMetricsRequest, GcpInfraPresenceRequest } from "@maple/domain/http" +import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" +import { WarehouseDateTimeString, decodeInput, runWarehouseQuery } from "@/api/warehouse/effect-utils" + +/** /infra/gcp data access over the `gcp.*` metrics the Google Cloud poller stores. */ + +const WindowInputSchema = Schema.Struct({ + startTime: WarehouseDateTimeString, + endTime: WarehouseDateTimeString, +}) + +const MetricsInputSchema = Schema.Struct({ + ...WindowInputSchema.fields, + service: Schema.Literals(GCP_INFRA_SERVICE_IDS), +}) + +/** The services with a metric in the window. */ +export const getGcpInfraPresence = Effect.fn("QueryEngine.getGcpInfraPresence")(function* ({ + data, +}: { + data: (typeof WindowInputSchema)["Encoded"] +}) { + const input = yield* decodeInput(WindowInputSchema, data, "getGcpInfraPresence") + const result = yield* runWarehouseQuery("gcpInfraPresence", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpInfraPresence({ + payload: new GcpInfraPresenceRequest(input), + }) + }), + ) + return { services: result.services } +}) + +/** One service's metrics: a point per workload, metric and label value. */ +export const getGcpInfraMetrics = Effect.fn("QueryEngine.getGcpInfraMetrics")(function* ({ + data, +}: { + data: (typeof MetricsInputSchema)["Encoded"] +}) { + const input = yield* decodeInput(MetricsInputSchema, data, "getGcpInfraMetrics") + const result = yield* runWarehouseQuery("gcpInfraMetrics", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpInfraMetrics({ + payload: new GcpInfraMetricsRequest(input), + }) + }), + ) + return { points: result.data } +}) diff --git a/apps/web/src/components/infra/gcp/tabs.test.ts b/apps/web/src/components/infra/gcp/tabs.test.ts new file mode 100644 index 0000000000..054f198255 --- /dev/null +++ b/apps/web/src/components/infra/gcp/tabs.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest" + +import { GCP_INFRA_SERVICE_IDS, gcpInfraMetrics } from "@maple/domain/gcp-infra" +import { GCP_ASSET_TYPES } from "@maple/domain/gcp-metrics" +import { + GCP_INFRA_COLUMNS, + GCP_NAME_SORT, + formatGcpValue, + gcpAssetTypeLabel, + gcpInfraNotice, + gcpInfraTabs, + gcpResourceName, + gcpResourcesError, + gcpWorkloads, + sortGcpWorkloads, + type GcpMetricPoint, + type GcpMetricReader, +} from "./tabs" + +const point = ( + keys: ReadonlyArray, + metric: string, + label: string, + total: number, + samples = 1, +): GcpMetricPoint => ({ keys, metric, label, total, samples }) + +const API = ["api", "acme-prod", "europe-west1"] +const WORKER = ["worker", "acme-prod", "europe-west1"] + +const valuesOf = (service: (typeof GCP_INFRA_SERVICE_IDS)[number], points: ReadonlyArray) => + gcpWorkloads(service, points).map((workload) => + Object.fromEntries(GCP_INFRA_COLUMNS[service].map((spec, index) => [spec.label, workload.values[index]])), + ) + +describe("GCP_INFRA_COLUMNS", () => { + it("reads only metrics the tab's query returns", () => { + for (const service of GCP_INFRA_SERVICE_IDS) { + const known = new Set(gcpInfraMetrics(service).map((metric) => metric.name)) + const read: Array = [] + const recorder: GcpMetricReader = { + total: (metric) => { + read.push(metric) + return 0 + }, + mean: (metric) => { + read.push(metric) + return undefined + }, + } + for (const spec of GCP_INFRA_COLUMNS[service]) spec.value(recorder) + expect(read.length, service).toBeGreaterThan(0) + expect( + read.filter((metric) => !known.has(metric)), + service, + ).toEqual([]) + } + }) + + it("labels every column uniquely in its tab, apart from the name sort key", () => { + for (const service of GCP_INFRA_SERVICE_IDS) { + const labels = GCP_INFRA_COLUMNS[service].map((spec) => spec.label) + expect(new Set([...labels, GCP_NAME_SORT]).size, service).toBe(labels.length + 1) + } + }) +}) + +describe("gcpWorkloads", () => { + it("folds a service's points into one row per workload", () => { + const rows = gcpWorkloads("cloudRun", [ + point(API, "gcp.run.request_count", "2xx", 900), + point(API, "gcp.run.request_count", "5xx", 100), + // 60 points whose mean is 120 ms. + point(API, "gcp.run.request_latencies", "0.95", 7200, 60), + point(API, "gcp.run.request_latencies", "0.99", 30_000, 60), + point(API, "gcp.run.container.instance_count", "active", 180, 60), + point(API, "gcp.run.container.instance_count", "idle", 60, 60), + point(API, "gcp.run.container.cpu.utilizations", "0.95", 30, 60), + point(WORKER, "gcp.run.container.instance_count", "active", 60, 60), + ]) + expect(rows.map((row) => row.keys)).toEqual([API, WORKER]) + expect(rows[0]?.values).toEqual([1000, 0.1, 120, 500, 3, 0.5, undefined]) + // A workload with no requests still has a row: counters read zero, the rest are absent. + expect(rows[1]?.values).toEqual([0, undefined, undefined, undefined, 1, undefined, undefined]) + }) + + it("counts the load balancer's numeric response class as a server error", () => { + const lb = ["web-map", "web-backend", "acme-prod"] + expect( + valuesOf("loadBalancing", [ + point(lb, "gcp.loadbalancing.https.request_count", "200", 75), + point(lb, "gcp.loadbalancing.https.request_count", "500", 25), + ])[0], + ).toMatchObject({ Requests: 100, "5xx rate": 0.25 }) + }) + + it("reads failures as everything outside the healthy label value", () => { + expect( + valuesOf("cloudFunctions", [ + point(API, "gcp.cloudfunctions.function.execution_count", "ok", 90), + point(API, "gcp.cloudfunctions.function.execution_count", "timeout", 6), + point(API, "gcp.cloudfunctions.function.execution_count", "crash", 4), + ])[0], + ).toMatchObject({ Executions: 100, "Error rate": 0.1 }) + // A pull subscription makes no push requests: there is no rate to show. + expect( + valuesOf("pubsub", [ + point(["orders", "acme-prod"], "gcp.pubsub.subscription.num_undelivered_messages", "", 0, 60), + ])[0], + ).toMatchObject({ Backlog: 0, Delivered: 0, "Push errors": undefined }) + }) + + it("turns CPU seconds per minute into cores and keeps to non-evictable memory", () => { + const container = ["api", "default", "prod", "acme-prod", "europe-west1"] + expect( + valuesOf("gke", [ + // 30 CPU seconds in each of 10 minutes. + point(container, "gcp.kubernetes.container.cpu.core_usage_time", "", 300, 10), + point(container, "gcp.kubernetes.container.memory.used_bytes", "non-evictable", 5120, 10), + point(container, "gcp.kubernetes.container.memory.used_bytes", "evictable", 99_999, 10), + ])[0], + ).toMatchObject({ "CPU cores": 0.5, Memory: 512, Restarts: 0, "CPU of limit": undefined }) + }) + + it("reads Cloud SQL connections from whichever engine metric reported", () => { + const db = ["acme-prod:main", "acme-prod", "europe-west1"] + const connections = (metric: string) => + valuesOf("cloudSql", [point(db, `gcp.cloudsql.database.${metric}`, "", 240, 60)])[0]?.Connections + expect(connections("network.connections")).toBe(4) + expect(connections("postgresql.num_backends")).toBe(4) + }) +}) + +describe("sortGcpWorkloads", () => { + const rows = gcpWorkloads("cloudRun", [ + point(["b"], "gcp.run.request_count", "2xx", 10), + point(["b"], "gcp.run.request_latencies", "0.95", 50), + point(["a"], "gcp.run.request_count", "2xx", 30), + point(["c"], "gcp.run.request_count", "2xx", 20), + point(["c"], "gcp.run.request_latencies", "0.95", 90), + ]) + const names = (key: string, dir: "asc" | "desc") => + sortGcpWorkloads("cloudRun", rows, { key, dir }).map((row) => row.keys[0]) + + it("sorts by a column in either direction", () => { + expect(names("Requests", "desc")).toEqual(["a", "c", "b"]) + expect(names("Requests", "asc")).toEqual(["b", "c", "a"]) + }) + + it("sorts by name", () => { + expect(names(GCP_NAME_SORT, "asc")).toEqual(["a", "b", "c"]) + expect(names(GCP_NAME_SORT, "desc")).toEqual(["c", "b", "a"]) + }) + + it("keeps a workload that never reported the column last in both directions", () => { + expect(names("Latency p95", "desc")).toEqual(["c", "b", "a"]) + expect(names("Latency p95", "asc")).toEqual(["b", "c", "a"]) + }) +}) + +describe("formatGcpValue", () => { + it("formats by unit and marks an absent value", () => { + expect(formatGcpValue("count", 12_345)).toBe("12.3K") + expect(formatGcpValue("percent", 0.42)).toBe("42%") + expect(formatGcpValue("errorRate", 0.001)).toBe("0.10%") + expect(formatGcpValue("ms", 1500)).toBe("1.50s") + expect(formatGcpValue("seconds", 90)).toBe("1.5min") + expect(formatGcpValue("bytes", 1536)).toBe("1.5 KB") + expect(formatGcpValue("bytes", undefined)).toBe("—") + }) +}) + +describe("gcpInfraTabs", () => { + it("shows the reporting services in tab order, then the inventory", () => { + expect(gcpInfraTabs(["cloudSql", "cloudRun"], undefined)).toEqual(["cloudRun", "cloudSql", "resources"]) + expect(gcpInfraTabs([], undefined)).toEqual(["resources"]) + }) + + it("keeps the requested tab when its service is quiet in the window", () => { + expect(gcpInfraTabs(["cloudRun"], "gke")).toEqual(["cloudRun", "gke", "resources"]) + expect(gcpInfraTabs(["cloudRun"], "resources")).toEqual(["cloudRun", "resources"]) + }) +}) + +describe("gcpInfraNotice", () => { + const connector = (last_metrics_received_at: string | null, last_metrics_error: string | null) => ({ + scope_type: "project" as const, + metrics_enabled: true, + last_metrics_received_at, + last_metrics_error, + discovered_project_count: 1, + last_resources_error: null, + }) + const READ_AT = "2026-10-08T09:10:00.000Z" + + it("says nothing while metrics arrive for the window", () => { + expect(gcpInfraNotice([connector(READ_AT, null)], true)).toBeNull() + }) + + it("waits for the first read, with the poller's reason", () => { + expect(gcpInfraNotice([connector(null, "No access yet.")], false)).toEqual({ + kind: "waiting", + note: "No access yet.", + }) + expect(gcpInfraNotice([connector(null, null)], false)).toEqual({ kind: "waiting", note: null }) + }) + + it("tells a quiet window from a connector that never read", () => { + expect(gcpInfraNotice([connector(null, null), connector(READ_AT, null)], false)).toEqual({ + kind: "quiet", + }) + }) + + it("reports a failing read even while other metrics arrive", () => { + expect( + gcpInfraNotice([connector(READ_AT, null), connector(READ_AT, "2 of 46 queries failed.")], true), + ).toEqual({ kind: "error", error: "2 of 46 queries failed." }) + }) +}) + +describe("resources", () => { + it("names every collected asset type", () => { + for (const assetType of GCP_ASSET_TYPES) { + expect(gcpAssetTypeLabel(assetType), assetType).not.toBe(assetType) + } + expect(gcpAssetTypeLabel("storage.googleapis.com/Bucket")).toBe("storage.googleapis.com/Bucket") + }) + + it("names a resource by its display name, else by the end of its full name", () => { + const name = "//run.googleapis.com/projects/acme-prod/locations/europe-west1/services/api" + expect(gcpResourceName({ name, displayName: "Public API" })).toBe("Public API") + expect(gcpResourceName({ name, displayName: null })).toBe("api") + expect(gcpResourceName({ name, displayName: "" })).toBe("api") + }) + + it("surfaces the first failing inventory sync", () => { + expect(gcpResourcesError([{ last_resources_error: null }])).toBeNull() + expect( + gcpResourcesError([{ last_resources_error: null }, { last_resources_error: "incomplete" }]), + ).toBe("incomplete") + }) +}) diff --git a/apps/web/src/components/infra/gcp/tabs.ts b/apps/web/src/components/infra/gcp/tabs.ts new file mode 100644 index 0000000000..405d365be8 --- /dev/null +++ b/apps/web/src/components/infra/gcp/tabs.ts @@ -0,0 +1,290 @@ +// Infrastructure -> Google Cloud: which tabs show, what each tab's columns read, and how the +// metric points of a tab fold into one table row per workload. Pure (no React, no atoms). + +import { GCP_INFRA_SERVICE_IDS, type GcpInfraServiceId } from "@maple/domain/gcp-infra" +import type { V2GcpConnector } from "@maple/domain/http/v2" +import { + EMPTY_VALUE, + formatBytes, + formatErrorRate, + formatLatency, + formatNumber, + formatPercent, +} from "@maple/ui/lib/format" +import { gcpMetricsState } from "@/components/integrations/gcp-connector-state" + +/** A row of the tab query: see `gcpInfraMetricsSQL`. */ +export interface GcpMetricPoint { + readonly keys: ReadonlyArray + readonly metric: string + readonly label: string + readonly total: number + readonly samples: number +} + +/** One workload's metrics over the window. Without `label`, every label value is added up. */ +export interface GcpMetricReader { + /** A counter's sum. The poller stores no zero deltas, so no points means zero. */ + readonly total: (metric: string, label?: string) => number + /** A gauge's or percentile's mean per point; undefined when it never reported. */ + readonly mean: (metric: string, label?: string) => number | undefined +} + +export type GcpColumnFormat = "count" | "decimal" | "percent" | "errorRate" | "ms" | "seconds" | "bytes" + +export interface GcpColumn { + /** Unique in its tab: also the column's sort key. */ + readonly label: string + readonly format: GcpColumnFormat + readonly value: (read: GcpMetricReader) => number | undefined +} + +export function formatGcpValue(format: GcpColumnFormat, value: number | undefined): string { + if (value === undefined) return EMPTY_VALUE + switch (format) { + case "count": + return formatNumber(value) + case "decimal": + return value.toLocaleString(undefined, { maximumFractionDigits: 2 }) + case "percent": + return formatPercent(value) + case "errorRate": + return formatErrorRate(value) + case "ms": + return formatLatency(value) + case "seconds": + return formatLatency(value * 1000) + case "bytes": + return formatBytes(value) + } +} + +const column = (label: string, format: GcpColumnFormat, value: GcpColumn["value"]): GcpColumn => ({ + label, + format, + value, +}) + +const share = (part: number, whole: number) => (whole > 0 ? part / whole : undefined) + +// Cloud Run labels a response class "5xx", the load balancer "500". +const serverErrorShare = (read: GcpMetricReader, metric: string) => + share(read.total(metric, "5xx") + read.total(metric, "500"), read.total(metric)) + +/** The share of a counter outside its one healthy label value. */ +const failedShare = (read: GcpMetricReader, metric: string, healthy: string) => + share(read.total(metric) - read.total(metric, healthy), read.total(metric)) + +const RUN = "gcp.run" +const FUNCTION = "gcp.cloudfunctions.function" +const CONTAINER = "gcp.kubernetes.container" +const INSTANCE = "gcp.compute.instance" +const DATABASE = "gcp.cloudsql.database" +const SUBSCRIPTION = "gcp.pubsub.subscription" +const HTTPS = "gcp.loadbalancing.https" + +export const GCP_INFRA_COLUMNS: Record> = { + cloudRun: [ + column("Requests", "count", (m) => m.total(`${RUN}.request_count`)), + column("5xx rate", "errorRate", (m) => serverErrorShare(m, `${RUN}.request_count`)), + column("Latency p95", "ms", (m) => m.mean(`${RUN}.request_latencies`, "0.95")), + column("Latency p99", "ms", (m) => m.mean(`${RUN}.request_latencies`, "0.99")), + column("Instances", "decimal", (m) => m.mean(`${RUN}.container.instance_count`, "active")), + column("CPU p95", "percent", (m) => m.mean(`${RUN}.container.cpu.utilizations`, "0.95")), + column("Memory p95", "percent", (m) => m.mean(`${RUN}.container.memory.utilizations`, "0.95")), + ], + cloudFunctions: [ + column("Executions", "count", (m) => m.total(`${FUNCTION}.execution_count`)), + column("Error rate", "errorRate", (m) => failedShare(m, `${FUNCTION}.execution_count`, "ok")), + column("Duration p95", "ms", (m) => m.mean(`${FUNCTION}.execution_times`, "0.95")), + column("Duration p99", "ms", (m) => m.mean(`${FUNCTION}.execution_times`, "0.99")), + column("Instances", "decimal", (m) => m.mean(`${FUNCTION}.instance_count`, "active")), + column("Memory p95", "bytes", (m) => m.mean(`${FUNCTION}.user_memory_bytes`, "0.95")), + ], + gke: [ + // A point is the CPU seconds used in one minute. + column("CPU cores", "decimal", (m) => { + const seconds = m.mean(`${CONTAINER}.cpu.core_usage_time`) + return seconds === undefined ? undefined : seconds / 60 + }), + column("CPU of limit", "percent", (m) => m.mean(`${CONTAINER}.cpu.limit_utilization`)), + // Non-evictable memory is what counts against the limit. + column("Memory", "bytes", (m) => m.mean(`${CONTAINER}.memory.used_bytes`, "non-evictable")), + column("Memory of limit", "percent", (m) => + m.mean(`${CONTAINER}.memory.limit_utilization`, "non-evictable"), + ), + column("Restarts", "count", (m) => m.total(`${CONTAINER}.restart_count`)), + ], + computeEngine: [ + column("CPU", "percent", (m) => m.mean(`${INSTANCE}.cpu.utilization`)), + // E2 machine types only. + column("Memory", "bytes", (m) => m.mean(`${INSTANCE}.memory.balloon.ram_used`)), + column("Network in", "bytes", (m) => m.total(`${INSTANCE}.network.received_bytes_count`)), + column("Network out", "bytes", (m) => m.total(`${INSTANCE}.network.sent_bytes_count`)), + column("Disk read", "bytes", (m) => m.total(`${INSTANCE}.disk.read_bytes_count`)), + column("Disk write", "bytes", (m) => m.total(`${INSTANCE}.disk.write_bytes_count`)), + ], + cloudSql: [ + column("CPU", "percent", (m) => m.mean(`${DATABASE}.cpu.utilization`)), + column("Memory", "percent", (m) => m.mean(`${DATABASE}.memory.utilization`)), + column("Disk", "percent", (m) => m.mean(`${DATABASE}.disk.utilization`)), + // MySQL and SQL Server report the first, PostgreSQL the second. + column( + "Connections", + "decimal", + (m) => m.mean(`${DATABASE}.network.connections`) ?? m.mean(`${DATABASE}.postgresql.num_backends`), + ), + column("Replica lag", "seconds", (m) => m.mean(`${DATABASE}.replication.replica_lag`)), + ], + pubsub: [ + column("Backlog", "decimal", (m) => m.mean(`${SUBSCRIPTION}.num_undelivered_messages`)), + column("Oldest unacked", "seconds", (m) => m.mean(`${SUBSCRIPTION}.oldest_unacked_message_age`)), + column("Delivered", "count", (m) => m.total(`${SUBSCRIPTION}.sent_message_count`)), + column("Acked", "count", (m) => m.total(`${SUBSCRIPTION}.ack_message_count`)), + column("Dead-lettered", "count", (m) => m.total(`${SUBSCRIPTION}.dead_letter_message_count`)), + // Push subscriptions only. + column("Push errors", "errorRate", (m) => failedShare(m, `${SUBSCRIPTION}.push_request_count`, "ack")), + ], + loadBalancing: [ + column("Requests", "count", (m) => m.total(`${HTTPS}.request_count`)), + column("5xx rate", "errorRate", (m) => serverErrorShare(m, `${HTTPS}.request_count`)), + column("Latency p95", "ms", (m) => m.mean(`${HTTPS}.total_latencies`, "0.95")), + column("Latency p99", "ms", (m) => m.mean(`${HTTPS}.total_latencies`, "0.99")), + column("Backend p95", "ms", (m) => m.mean(`${HTTPS}.backend_latencies`, "0.95")), + column("Response bytes", "bytes", (m) => m.total(`${HTTPS}.response_bytes_count`)), + ], +} satisfies Record> + +const readerOf = (points: ReadonlyArray): GcpMetricReader => { + const matching = (metric: string, label?: string) => + points.filter((point) => point.metric === metric && (label === undefined || point.label === label)) + return { + total: (metric, label) => matching(metric, label).reduce((sum, point) => sum + point.total, 0), + mean: (metric, label) => { + const matched = matching(metric, label) + return matched.length === 0 + ? undefined + : matched.reduce((sum, point) => sum + point.total / point.samples, 0) + }, + } +} + +export interface GcpWorkload { + /** Identity values, in the order of the service's `identity`. The first names the row. */ + readonly keys: ReadonlyArray + /** One value per column of the tab; undefined where the workload reported nothing. */ + readonly values: ReadonlyArray +} + +/** One table row per workload, from the tab query's points. */ +export function gcpWorkloads( + service: GcpInfraServiceId, + points: ReadonlyArray, +): ReadonlyArray { + const byWorkload = new Map; points: Array }>() + for (const point of points) { + const id = point.keys.join("\u0000") + const workload = byWorkload.get(id) + if (workload === undefined) byWorkload.set(id, { keys: point.keys, points: [point] }) + else workload.points.push(point) + } + return [...byWorkload.values()].map((workload) => { + const read = readerOf(workload.points) + return { keys: workload.keys, values: GCP_INFRA_COLUMNS[service].map((spec) => spec.value(read)) } + }) +} + +/** Sort key of the name column; every other key is a column label. */ +export const GCP_NAME_SORT = "name" + +/** Sorted by the name or by one column. A value that never reported sorts last either way. */ +export function sortGcpWorkloads( + service: GcpInfraServiceId, + workloads: ReadonlyArray, + sort: { readonly key: string; readonly dir: "asc" | "desc" }, +): ReadonlyArray { + const sign = sort.dir === "asc" ? 1 : -1 + const index = GCP_INFRA_COLUMNS[service].findIndex((spec) => spec.label === sort.key) + return [...workloads].sort((a, b) => { + if (sort.key === GCP_NAME_SORT) return sign * (a.keys[0] ?? "").localeCompare(b.keys[0] ?? "") + const left = a.values[index] + const right = b.values[index] + if (left === undefined || right === undefined) { + return left === right ? 0 : left === undefined ? 1 : -1 + } + return sign * (left - right) + }) +} + +export const GCP_RESOURCES_TAB = "resources" +export const GCP_INFRA_TABS = [...GCP_INFRA_SERVICE_IDS, GCP_RESOURCES_TAB] as const +export type GcpInfraTab = (typeof GCP_INFRA_TABS)[number] + +/** + * The tabs on the page: the services reporting in the window, then the inventory. A requested + * service tab stays, so a link or a narrower time range lands on its empty state. + */ +export const gcpInfraTabs = ( + reporting: ReadonlyArray, + requested: GcpInfraTab | undefined, +): ReadonlyArray => + GCP_INFRA_TABS.filter( + (tab) => tab === GCP_RESOURCES_TAB || tab === requested || reporting.includes(tab), + ) + +export type GcpInfraNotice = + /** A connector's most recent metrics read failed or was incomplete. */ + | { readonly kind: "error"; readonly error: string } + /** No connector has read metrics yet. `note` is the poller's reason, if it gave one. */ + | { readonly kind: "waiting"; readonly note: string | null } + /** Metrics arrive, but none in the selected window. */ + | { readonly kind: "quiet" } + +type MetricsFields = Parameters[0] + +/** What to say above the tabs, given the connectors that collect metrics. */ +export function gcpInfraNotice( + connectors: ReadonlyArray, + reporting: boolean, +): GcpInfraNotice | null { + const states = connectors.map(gcpMetricsState) + for (const state of states) { + if (state.kind === "error") return { kind: "error", error: state.error } + } + if (reporting) return null + if (states.some((state) => state.kind === "receiving")) return { kind: "quiet" } + for (const state of states) { + if (state.kind === "waiting") return { kind: "waiting", note: state.note } + } + return null +} + +/** Why the inventory may be stale: the first failing resource sync, or null. */ +export const gcpResourcesError = ( + connectors: ReadonlyArray>, +): string | null => connectors.find((connector) => connector.last_resources_error !== null)?.last_resources_error ?? null + +const ASSET_TYPE_LABELS: Record = { + "cloudresourcemanager.googleapis.com/Project": "Project", + "run.googleapis.com/Service": "Cloud Run service", + "run.googleapis.com/Job": "Cloud Run job", + "cloudfunctions.googleapis.com/CloudFunction": "Cloud Function (1st gen)", + "cloudfunctions.googleapis.com/Function": "Cloud Function (2nd gen)", + "container.googleapis.com/Cluster": "GKE cluster", + "compute.googleapis.com/Instance": "VM instance", + "sqladmin.googleapis.com/Instance": "Cloud SQL instance", + "pubsub.googleapis.com/Topic": "Pub/Sub topic", + "pubsub.googleapis.com/Subscription": "Pub/Sub subscription", + "compute.googleapis.com/UrlMap": "URL map", + "compute.googleapis.com/BackendService": "Backend service", + "compute.googleapis.com/ForwardingRule": "Forwarding rule", +} satisfies Record + +/** A Cloud Asset Inventory type in words; the raw type when it is not one Maple collects. */ +export const gcpAssetTypeLabel = (assetType: string): string => ASSET_TYPE_LABELS[assetType] ?? assetType + +/** What to call a resource: its display name, else the last segment of its full name. */ +export const gcpResourceName = (resource: { + readonly name: string + readonly displayName: string | null +}): string => resource.displayName || (resource.name.split("/").at(-1) ?? resource.name) diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index e723d49af5..ee09ed9ba2 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -85,6 +85,7 @@ import { getPlanetScaleQueryInsights, } from "@/api/warehouse/planetscale-infra" import { getRailwayServices, getRailwayServiceTimeseries } from "@/api/warehouse/railway-infra" +import { getGcpInfraMetrics, getGcpInfraPresence } from "@/api/warehouse/gcp-infra" import { getServiceHealthBaseline, getServiceHealthSnapshot, @@ -788,6 +789,14 @@ export const railwayServiceTimeseriesResultAtom = makeQueryAtomFamily(getRailway staleTime: 15_000, }) +export const gcpInfraPresenceResultAtom = makeQueryAtomFamily(getGcpInfraPresence, { + staleTime: 60_000, +}) + +export const gcpInfraMetricsResultAtom = makeQueryAtomFamily(getGcpInfraMetrics, { + staleTime: 15_000, +}) + export const planetscaleQueryInsightsResultAtom = makeQueryAtomFamily(getPlanetScaleQueryInsights, { // Server-side edge cache is 60s; match it so refreshes don't hammer PlanetScale. staleTime: 60_000, diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts index dda42d9d1d..28d16192f7 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.test.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -278,6 +278,87 @@ describe("GcpConnectorService", () => { }).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))) }) + it.effect("lists the organization's inventory once per resource, filtered, with facets over all of it", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + const wide = yield* gcp.create(orgId, userId, { ...organizationScope, metricsEnabled: false }) + // A project inside the organization: both connectors list its resources. + const narrow = yield* gcp.create(orgId, userId, projectScope("acme-shop")) + const foreign = yield* gcp.create(otherOrgId, userId, projectScope("elsewhere")) + + const PROJECT = "cloudresourcemanager.googleapis.com/Project" + const RUN = "run.googleapis.com/Service" + const SQL = "sqladmin.googleapis.com/Instance" + const api = "//run.googleapis.com/projects/acme-shop/locations/europe-west1/services/api" + const seen: ReadonlyArray = [ + [wide, orgId, "//cloudresourcemanager.googleapis.com/projects/111", PROJECT, "acme-shop"], + [wide, orgId, "//cloudresourcemanager.googleapis.com/projects/222", PROJECT, "acme-blog"], + [wide, orgId, api, RUN, "acme-shop"], + [wide, orgId, "//sqladmin.googleapis.com/projects/acme-blog/instances/db", SQL, "acme-blog"], + [narrow, orgId, "//cloudresourcemanager.googleapis.com/projects/111", PROJECT, "acme-shop"], + [narrow, orgId, api, RUN, "acme-shop"], + [foreign, otherOrgId, "//run.googleapis.com/projects/elsewhere/services/x", RUN, "elsewhere"], + ] + for (const [connector, org, name, assetType, projectId] of seen) { + yield* Effect.promise(() => + executeSql( + testDb, + `INSERT INTO gcp_resources + (connector_id, org_id, name, asset_type, project_id, location, display_name, state, labels, last_seen_at) + VALUES ($1, $2, $3, $4, $5, 'europe-west1', 'shown', 'ACTIVE', '{"team":"core"}', now())`, + [connector.id, org, name, assetType, projectId], + ), + ) + } + + const all = yield* gcp.resources(orgId, {}) + assert.strictEqual(all.total, 4) + assert.deepStrictEqual( + all.resources.map((resource) => resource.name), + [ + "//cloudresourcemanager.googleapis.com/projects/111", + "//cloudresourcemanager.googleapis.com/projects/222", + api, + "//sqladmin.googleapis.com/projects/acme-blog/instances/db", + ], + ) + assert.deepStrictEqual(all.types, [ + { assetType: PROJECT, count: 2 }, + { assetType: RUN, count: 1 }, + { assetType: SQL, count: 1 }, + ]) + assert.deepStrictEqual(all.projects, ["acme-blog", "acme-shop"]) + + const services = yield* gcp.resources(orgId, { assetType: RUN }) + assert.deepStrictEqual(services.resources, [ + { + name: api, + assetType: RUN, + projectId: "acme-shop", + location: "europe-west1", + displayName: "shown", + state: "ACTIVE", + labels: { team: "core" }, + }, + ]) + assert.strictEqual(services.total, 1) + // The filter's options do not narrow with it. + assert.deepStrictEqual(services.types, all.types) + assert.deepStrictEqual(services.projects, all.projects) + + assert.strictEqual((yield* gcp.resources(orgId, { projectId: "acme-blog" })).total, 2) + const none = yield* gcp.resources(orgId, { assetType: RUN, projectId: "acme-blog" }) + assert.deepStrictEqual([none.total, none.resources], [0, []]) + + const theirs = yield* gcp.resources(otherOrgId, {}) + assert.deepStrictEqual( + theirs.resources.map((resource) => resource.projectId), + ["elsewhere"], + ) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + it.effect("refuses to render a secret that was moved onto another connector's row", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index 02af1895e8..59bc7e81e4 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -8,7 +8,9 @@ import { } from "@maple/db" import { GCP_PROJECT_ASSET_TYPE } from "@maple/domain/gcp-metrics" import { + GCP_RESOURCES_LIMIT, GcpMetricsUnavailableError, + type GcpResource, GcpScopeAlreadyConnectedError, IntegrationsNotFoundError, IntegrationsPersistenceError, @@ -60,6 +62,22 @@ export type CreateGcpConnectorInput = Pick< type GcpCapabilityError = GcpMetricsUnavailableError | IntegrationsValidationError +export interface GcpResourceFilter { + readonly assetType?: string | undefined + readonly projectId?: string | undefined +} + +export interface GcpResourceInventory { + /** The first `GCP_RESOURCES_LIMIT` resources matching the filter, by asset type and name. */ + readonly resources: ReadonlyArray + /** How many resources match the filter. */ + readonly total: number + /** The whole inventory by asset type. */ + readonly types: ReadonlyArray<{ readonly assetType: string; readonly count: number }> + /** Every project with a resource. */ + readonly projects: ReadonlyArray +} + export interface GcpConnectorServiceApi { readonly status: (orgId: OrgId) => Effect.Effect< { @@ -94,6 +112,11 @@ export interface GcpConnectorServiceApi { { readonly setupScript: string; readonly cleanupScript: string }, IntegrationsNotFoundError | IntegrationsPersistenceError > + /** What the latest inventory syncs of the org's connectors found. */ + readonly resources: ( + orgId: OrgId, + filter: GcpResourceFilter, + ) => Effect.Effect /** Deleting the row is the disconnect: the ingest gateway stops accepting the connector's pushes. */ readonly delete: ( orgId: OrgId, @@ -218,6 +241,74 @@ export class GcpConnectorService extends Context.Service + db + .select({ + assetType: gcpResources.assetType, + projectId: gcpResources.projectId, + count: sql`count(distinct ${gcpResources.name})::int`, + }) + .from(gcpResources) + .innerJoin(gcpConnectors, eq(gcpResources.connectorId, gcpConnectors.id)) + .where(ofOrg) + .groupBy(gcpResources.assetType, gcpResources.projectId), + ) + const rows = yield* dbExecute((db) => + db + .selectDistinctOn([gcpResources.assetType, gcpResources.name], { + name: gcpResources.name, + assetType: gcpResources.assetType, + projectId: gcpResources.projectId, + location: gcpResources.location, + displayName: gcpResources.displayName, + state: gcpResources.state, + labels: gcpResources.labels, + }) + .from(gcpResources) + .innerJoin(gcpConnectors, eq(gcpResources.connectorId, gcpConnectors.id)) + .where( + and( + ofOrg, + filter.assetType === undefined + ? undefined + : eq(gcpResources.assetType, filter.assetType), + filter.projectId === undefined + ? undefined + : eq(gcpResources.projectId, filter.projectId), + ), + ) + .orderBy(asc(gcpResources.assetType), asc(gcpResources.name)) + .limit(GCP_RESOURCES_LIMIT), + ) + const byType = new Map() + let total = 0 + for (const { assetType, projectId, count } of counts) { + byType.set(assetType, (byType.get(assetType) ?? 0) + count) + if ( + (filter.assetType === undefined || filter.assetType === assetType) && + (filter.projectId === undefined || filter.projectId === projectId) + ) { + total += count + } + } + return { + resources: rows, + total, + types: [...byType] + .map(([assetType, count]) => ({ assetType, count })) + .sort((a, b) => a.assetType.localeCompare(b.assetType)), + projects: [...new Set(counts.map((row) => row.projectId))].sort(), + } + }) + const create = Effect.fn("GcpConnectorService.create")(function* ( orgId: OrgId, userId: UserId, @@ -371,6 +462,7 @@ export class GcpConnectorService extends Context.Service + +export class GcpResourcesResponse extends Schema.Class("GcpResourcesResponse")({ + /** The first `GCP_RESOURCES_LIMIT` resources matching the filter, by asset type and name. */ + resources: Schema.Array(GcpResource), + /** How many resources match the filter. */ + total: Schema.Number, + /** The whole inventory by asset type: the type filter's options, and the project count. */ + types: Schema.Array(Schema.Struct({ assetType: Schema.String, count: Schema.Number })), + /** Every project with a resource: the project filter's options. */ + projects: Schema.Array(Schema.String), +}) {} + +const GcpResourceFilterValue = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256)) + export class RailwayEnvironmentStatus extends Schema.Class( "RailwayEnvironmentStatus", )({ @@ -1231,6 +1260,18 @@ export class IntegrationsApiGroup extends HttpApiGroup.make("integrations") error: [IntegrationsForbiddenError, IntegrationsPersistenceError], }), ) + .add( + // The inventory behind Infrastructure -> Google Cloud. Connectors are public v2 + // (`/v2/integrations/gcp`); only the dashboard reads what they discovered. + HttpApiEndpoint.get("gcpResources", "/gcp/resources", { + query: Schema.Struct({ + assetType: Schema.optional(GcpResourceFilterValue), + projectId: Schema.optional(GcpResourceFilterValue), + }), + success: GcpResourcesResponse, + error: IntegrationsPersistenceError, + }), + ) .add( HttpApiEndpoint.get("githubStatus", "/github/status", { success: GithubIntegrationStatus, diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 96123840dd..2a931b4a3f 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -1,5 +1,6 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" import { Schema } from "effect" +import { GCP_INFRA_SERVICE_IDS } from "../gcp-infra" import { MAX_RAW_SQL_LENGTH } from "../raw-sql" import { RawSqlDisplayType } from "@maple/widgets" import { @@ -455,6 +456,44 @@ export class RailwayInfraRowsResponse extends Schema.Class( + "GcpInfraPresenceRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, +}) {} + +export class GcpInfraPresenceResponse extends Schema.Class( + "GcpInfraPresenceResponse", +)({ + /** The services with a metric in the window, in tab order. */ + services: Schema.Array(GcpInfraService), +}) {} + +export class GcpInfraMetricsRequest extends Schema.Class("GcpInfraMetricsRequest")({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + service: GcpInfraService, +}) {} + +export class GcpInfraMetricsResponse extends Schema.Class( + "GcpInfraMetricsResponse", +)({ + /** One row per workload, metric and label value: see `gcpInfraMetricsSQL`. */ + data: Schema.Array( + Schema.Struct({ + keys: Schema.Array(Schema.String), + metric: Schema.String, + label: Schema.String, + total: Schema.Number, + samples: Schema.Number, + }), + ), +}) {} + // Cloudflare infrastructure page (/infra/cloudflare): per-zone HTTP edge // analytics and per-Worker invocation analytics from the direct-integration // poller's metrics. Same conventions as ServiceCloudflareStats — no @@ -2626,6 +2665,20 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("gcpInfraPresence", "/gcp-infra-presence", { + payload: GcpInfraPresenceRequest, + success: GcpInfraPresenceResponse, + error: queryEngineEndpointErrors, + }), + ) + .add( + HttpApiEndpoint.post("gcpInfraMetrics", "/gcp-infra-metrics", { + payload: GcpInfraMetricsRequest, + success: GcpInfraMetricsResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("cloudflareInfraZones", "/cloudflare-infra-zones", { payload: CloudflareInfraZonesRequest, diff --git a/packages/query-engine-integrations/src/gcp/gcp-infra.ts b/packages/query-engine-integrations/src/gcp/gcp-infra.ts index 9295ce44dc..80d2958860 100644 --- a/packages/query-engine-integrations/src/gcp/gcp-infra.ts +++ b/packages/query-engine-integrations/src/gcp/gcp-infra.ts @@ -26,21 +26,11 @@ type MetricsAccessor = | ColumnAccessor | ColumnAccessor -export interface GcpInfraMetricsOutput { - /** The workload's identity values, in the order of the service's `identity`. */ - readonly keys: ReadonlyArray - readonly metric: string - /** The metric's label value or quantile; empty when it has neither. */ - readonly label: string - /** Sum of the points in the window. */ - readonly total: number - /** Number of points in the window. */ - readonly samples: number -} - /** - * Every curated metric of one service: one row per workload, metric and label value. A counter - * reads as `total`, a gauge or percentile as the mean `total / samples`. + * Every curated metric of one service: one row per workload (`keys`, the values of the service's + * `identity`), metric and `label` (the metric's label value or quantile, empty without one). + * `total` sums the window's points and `samples` counts them: a counter reads as `total`, a + * gauge or percentile as the mean `total / samples`. */ export function gcpInfraMetricsSQL(service: GcpInfraServiceId) { const metrics = gcpInfraMetrics(service) @@ -92,10 +82,6 @@ export function gcpInfraMetricsSQL(service: GcpInfraServiceId) { .format("JSON") } -export interface GcpInfraPresenceOutput { - readonly metric: string -} - /** Which of the page's metrics reported in the window, from the hourly `metric_catalog` rollup. */ export function gcpInfraPresenceSQL() { return from(MetricCatalog) diff --git a/packages/query-engine-integrations/src/gcp/index.ts b/packages/query-engine-integrations/src/gcp/index.ts index 30a5987a30..71d329fdd6 100644 --- a/packages/query-engine-integrations/src/gcp/index.ts +++ b/packages/query-engine-integrations/src/gcp/index.ts @@ -1,8 +1,3 @@ // Google Cloud integration queries: the /infra/gcp service tabs. -export { - gcpInfraMetricsSQL, - gcpInfraPresenceSQL, - type GcpInfraMetricsOutput, - type GcpInfraPresenceOutput, -} from "./gcp-infra" +export { gcpInfraMetricsSQL, gcpInfraPresenceSQL } from "./gcp-infra" From da440caea5645b7634d6ca5abb75048b6609f08d Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 21:58:28 +0200 Subject: [PATCH 03/51] feat(web): Infrastructure -> Google Cloud page --- .../src/content/docs/integrations/gcp.md | 21 ++ .../src/components/dashboard/app-sidebar.tsx | 5 +- .../components/dashboard/nav-items.test.ts | 13 +- .../web/src/components/dashboard/nav-items.ts | 11 +- .../components/infra/gcp/gcp-resources.tsx | 228 ++++++++++++++ .../infra/gcp/gcp-service-table.tsx | 167 ++++++++++ .../web/src/components/infra/gcp/tabs.test.ts | 10 +- apps/web/src/components/infra/gcp/tabs.ts | 11 +- apps/web/src/hooks/use-infra-surfaces.ts | 19 +- apps/web/src/routeTree.gen.ts | 21 ++ apps/web/src/routes/infra/gcp/index.tsx | 284 ++++++++++++++++++ docs/api-v2.md | 2 +- .../integrations/GcpConnectorService.test.ts | 175 ++++++----- packages/domain/src/http/query-engine.ts | 40 +-- .../src/__sql_baseline__/integrations.sql | 256 ++++++++++++++++ .../src/benchmark/index.ts | 14 +- .../src/gcp/gcp-infra.ts | 4 +- 17 files changed, 1150 insertions(+), 131 deletions(-) create mode 100644 apps/web/src/components/infra/gcp/gcp-resources.tsx create mode 100644 apps/web/src/components/infra/gcp/gcp-service-table.tsx create mode 100644 apps/web/src/routes/infra/gcp/index.tsx diff --git a/apps/landing/src/content/docs/integrations/gcp.md b/apps/landing/src/content/docs/integrations/gcp.md index 5cd222c535..5944a91843 100644 --- a/apps/landing/src/content/docs/integrations/gcp.md +++ b/apps/landing/src/content/docs/integrations/gcp.md @@ -67,6 +67,7 @@ Then check the data: 1. Open [Logs](/docs/explore/logs) and look for a service named after one of your Cloud Run services, functions, containers or instances. 2. In the [metrics explorer](/docs/explore/metrics), search for `gcp.`. 3. Under **Dashboards → Browse templates**, create the **Google Cloud** dashboard. It takes an optional project ID. +4. Open **Infrastructure → Google Cloud** for a table of your workloads per service. ## Permissions @@ -182,6 +183,25 @@ Every series also carries `cloud.account.id` (the project ID) and, for resources Every hour Maple lists these resources from Cloud Asset Inventory, with their project, location, state and labels: projects, Cloud Run services and jobs, Cloud Functions, GKE clusters, Compute Engine instances, Cloud SQL instances, Pub/Sub topics and subscriptions, and load balancer URL maps, backend services and forwarding rules. +## Infrastructure → Google Cloud + +**Infrastructure → Google Cloud** appears in the sidebar once a connection has **Metrics and resources** switched on. It has one tab per service that reported metrics in the selected time range, with one row per workload. Counts and byte totals cover the time range; everything else is the average over it. + +| Tab | One row per | Columns | +| --------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| Cloud Run | Service, project and region | Requests, 5xx rate, latency p95 and p99, active instances, CPU p95, memory p95 | +| Cloud Functions | 1st gen function, project and region | Executions, error rate, duration p95 and p99, active instances, memory p95 | +| GKE | Container name, namespace and cluster | CPU cores, CPU of limit, memory, memory of limit, restarts | +| Compute Engine | Instance, project and zone | CPU, memory (E2 machine types only), network in and out, disk read and write | +| Cloud SQL | Instance, project and region | CPU, memory, disk, connections, replica lag | +| Pub/Sub | Subscription and project | Backlog, age of the oldest unacknowledged message, delivered, acknowledged, dead-lettered, push errors | +| Load Balancing | URL map, backend and project | Requests, 5xx rate, latency p95 and p99, backend latency p95, response bytes | +| Resources | Resource from the [inventory](#resources) | Type, project, location, state and labels, with a filter by type and by project | + +Search a tab by name, project or location, and click a column to sort by it. A dash means the workload did not report that metric. Cloud Functions (2nd gen) run on Cloud Run and appear on the Cloud Run tab. GKE node and Pub/Sub topic metrics are collected but have no tab: chart them in a dashboard. + +The **Resources** tab shows the first 500 resources that match its filters, and how many projects Maple found. It does not depend on the time range. + ## Google Cloud costs Google bills Pub/Sub and Cloud Monitoring usage to your account, separately from your Maple plan. @@ -249,5 +269,6 @@ Until the cleanup script runs, the sink keeps publishing to Pub/Sub. The script - [Logs](/docs/explore/logs): search Google Cloud logs next to your application logs. - [Dashboards](/docs/dashboards/build-dashboards#templates): start from the **Google Cloud** template. +- **Infrastructure → Google Cloud**: scan every workload of a service in one table. - [Alert rules](/docs/alerting/alert-rules): alert on any `gcp.*` metric. - [API reference](/docs/reference/api): manage connections with the `/v2/integrations/gcp` endpoints. diff --git a/apps/web/src/components/dashboard/app-sidebar.tsx b/apps/web/src/components/dashboard/app-sidebar.tsx index 30e9a7c94a..7c6b2162f5 100644 --- a/apps/web/src/components/dashboard/app-sidebar.tsx +++ b/apps/web/src/components/dashboard/app-sidebar.tsx @@ -298,7 +298,8 @@ function NavRow({ const seen = new Set() const unique: NavSubItem[] = [] for (const sub of item.subItems) { - if (!sub.icon || seen.has(sub.icon)) continue + // The section's own front door (Infrastructure's Overview) is not one of its contents. + if (!sub.icon || sub.href === item.href || seen.has(sub.icon)) continue seen.add(sub.icon) unique.push(sub) } @@ -307,7 +308,7 @@ function NavRow({ // preview is all or nothing. Six fits both sections we ship (Explore // with Agent Sessions on, Infrastructure) at the tightened gap below. return unique.length > NAV_PREVIEW_MAX_GLYPHS ? undefined : unique - }, [isOpen, item.subItems]) + }, [isOpen, item.subItems, item.href]) // The sub-list can't render at 48px, so the rail turns the row into a menu. // Without this, every child route is stranded while the sidebar is collapsed diff --git a/apps/web/src/components/dashboard/nav-items.test.ts b/apps/web/src/components/dashboard/nav-items.test.ts index cd9272b0d4..e4d301c5ee 100644 --- a/apps/web/src/components/dashboard/nav-items.test.ts +++ b/apps/web/src/components/dashboard/nav-items.test.ts @@ -124,11 +124,12 @@ describe("navGroups", () => { } }) - it("keeps Infrastructure at six children with six unique glyphs", () => { + it("keeps Infrastructure at six sources with six unique glyphs", () => { // Six is exactly NavRow's all-or-nothing preview cap, so a seventh unique // glyph here would drop the closed row's miniatures entirely rather than // truncate them. A new Kubernetes view goes in `views` (free); a new - // child with a new glyph is not free. + // child with a new glyph is not free. Overview is the section's own page + // and stays out of the preview. const infra = findItem("Infrastructure") expect(infra.subItems?.map((sub) => sub.title)).toEqual([ "Overview", @@ -137,8 +138,10 @@ describe("navGroups", () => { "Cloudflare", "PlanetScale", "Railway", + "Google Cloud", ]) - expect(new Set(infra.subItems?.map((sub) => sub.icon)).size).toBe(NAV_PREVIEW_MAX_GLYPHS) + const sources = infra.subItems?.filter((sub) => sub.href !== infra.href) + expect(new Set(sources?.map((sub) => sub.icon)).size).toBe(NAV_PREVIEW_MAX_GLYPHS) }) it("folds the Kubernetes views behind one row that points at the section root", () => { @@ -214,7 +217,7 @@ describe("partitionInfraSubItems", () => { it("shows every child while the org's surfaces are unknown", () => { const { shown, hidden } = partitionInfraSubItems(subItems(), null, "/infra") - expect(shown).toHaveLength(6) + expect(shown).toHaveLength(7) expect(hidden).toEqual([]) }) @@ -225,7 +228,7 @@ describe("partitionInfraSubItems", () => { "/infra", ) expect(titles(shown)).toEqual(["Overview", "Hosts", "PlanetScale"]) - expect(titles(hidden)).toEqual(["Kubernetes", "Cloudflare", "Railway"]) + expect(titles(hidden)).toEqual(["Kubernetes", "Cloudflare", "Railway", "Google Cloud"]) }) // Containers is a view of Hosts, so a Docker-only org still gets the row. diff --git a/apps/web/src/components/dashboard/nav-items.ts b/apps/web/src/components/dashboard/nav-items.ts index 4c6dad4189..5e9f2aa09c 100644 --- a/apps/web/src/components/dashboard/nav-items.ts +++ b/apps/web/src/components/dashboard/nav-items.ts @@ -7,6 +7,7 @@ import { CodeIcon, ComputerIcon, FileIcon, + GoogleCloudIcon, GridIcon, GridSquareCirclePlusIcon, HouseIcon, @@ -27,8 +28,9 @@ import type { OrganizationFeatureFlags } from "@/lib/organization-feature-flags" /** * What a nav child needs from the org before it's worth a row. The five OTel - * surfaces come from the warehouse presence probe; Cloudflare, PlanetScale and - * Railway are integration pages, so their gate is whether the integration is connected. + * surfaces come from the warehouse presence probe; Cloudflare, PlanetScale, + * Railway and Google Cloud are integration pages, so their gate is whether the + * integration is connected (for Google Cloud: with metrics switched on). */ export type NavSurface = | "hosts" @@ -39,6 +41,7 @@ export type NavSurface = | "cloudflare" | "planetscale" | "railway" + | "gcp" export interface NavSubItem { title: string @@ -101,7 +104,8 @@ const overviewItem: NavItem = { * connected; the ones it doesn't have are offered on the Overview instead of * padded into the nav. Containers folds into Hosts as a view, the way Pods, * Nodes and Workloads fold into Kubernetes, and stays typeable in the palette - * through `views`. Six children with six unique marks fits `NavRow`'s preview cap. + * through `views`. The six sources' marks are exactly `NavRow`'s preview cap, which + * leaves the Overview row's own glyph out. */ const infrastructureItem: NavItem = { title: "Infrastructure", @@ -134,6 +138,7 @@ const infrastructureItem: NavItem = { surfaces: ["planetscale"], }, { title: "Railway", href: "/infra/railway", icon: RailwayIcon, surfaces: ["railway"] }, + { title: "Google Cloud", href: "/infra/gcp", icon: GoogleCloudIcon, surfaces: ["gcp"] }, ], } diff --git a/apps/web/src/components/infra/gcp/gcp-resources.tsx b/apps/web/src/components/infra/gcp/gcp-resources.tsx new file mode 100644 index 0000000000..af9cf9ead3 --- /dev/null +++ b/apps/web/src/components/infra/gcp/gcp-resources.tsx @@ -0,0 +1,228 @@ +import { GCP_PROJECT_ASSET_TYPE } from "@maple/domain/gcp-metrics" +import type { GcpResourcesResponse } from "@maple/domain/http" +import { Alert, AlertDescription, AlertTitle } from "@maple/ui/components/ui/alert" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@maple/ui/components/ui/select" +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { EMPTY_VALUE, countLabel } from "@maple/ui/lib/format" + +import { ColumnHead, DataTable, MetaChip } from "@/components/common/data-table" +import { ResultView } from "@/components/common/result-view" +import { CircleWarningIcon } from "@/components/icons" +import { useRefreshableAtomValue } from "@/hooks/use-refreshable-atom-value" +import { retainedInternalQuery } from "@/lib/services/common/internal-atom-client" + +import { gcpAssetTypeLabel, gcpResourceName } from "./tabs" + +export interface GcpResourceFilter { + readonly type?: string | undefined + readonly project?: string | undefined +} + +const ALL = "all" +const MAX_LABELS = 3 + +function FilterSelect({ + label, + allLabel, + options, + value, + onChange, +}: { + label: string + allLabel: string + options: ReadonlyArray<{ value: string; label: string }> + value: string | undefined + onChange: (value: string | undefined) => void +}) { + const items = [{ value: ALL, label: allLabel }, ...options] + return ( + + ) +} + +function Columns() { + return ( + <> + + + + + +