diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7054601a9..740731d4a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,8 @@ jobs: - 'packages/backend/src/services/warehouse/**' - 'packages/db/**' - 'packages/query-engine/**' + # The sweep analyzes the integration builders too. + - 'packages/query-engine-integrations/**' - 'packages/domain/src/clickhouse/**' - 'packages/domain/src/tinybird/**' - 'packages/domain/src/generated/**' diff --git a/apps/api/src/routes/internal/integrations.http.ts b/apps/api/src/routes/internal/integrations.http.ts index 7266af0c44..6abd5cb91d 100644 --- a/apps/api/src/routes/internal/integrations.http.ts +++ b/apps/api/src/routes/internal/integrations.http.ts @@ -10,6 +10,7 @@ import { CloudflareUsageResponse, CurrentTenant, ExternalUserId, + GcpResourcesResponse, GithubDeleteRepositoryResponse, GithubDisconnectResponse, GithubIntegrationStatus, @@ -54,6 +55,7 @@ import { topTrafficQuery, type TopTrafficGroupDefinition, } from "@maple/backend/services/integrations/cloudflare-analytics/queries" +import { GcpConnectorService } from "@maple/backend/services/integrations/GcpConnectorService" import { RailwayMetricsService } from "@maple/backend/services/integrations/RailwayMetricsService" import { GithubConnectService } from "@maple/backend/services/integrations/vcs/vendor/github/GithubConnectService" import { VcsCommitService } from "@maple/backend/services/integrations/vcs/VcsCommitService" @@ -118,6 +120,7 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte const cloudflare = yield* CloudflareOAuthService const cloudflareAnalytics = yield* CloudflareAnalyticsService const railway = yield* RailwayMetricsService + const gcp = yield* GcpConnectorService const edgeCache = yield* EdgeCacheService const env = yield* Env @@ -464,6 +467,12 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte return new RailwayDisconnectResponse(yield* railway.disconnect(tenant.orgId)) }), ) + .handle("gcpResources", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + return new GcpResourcesResponse(yield* gcp.resources(tenant.orgId, query)) + }), + ) .handle("githubStatus", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 9b5fb46bdc..7f73fcb1ab 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -19,6 +19,12 @@ import { ServiceApdexResponse, PlanetScaleInfraTimeseriesResponse, RailwayInfraRowsResponse, + GcpInfraMetricsResponse, + GcpInfraPresenceResponse, + GcpInfraSparklinesResponse, + GcpCloudSqlDatabasesResponse, + GcpCloudSqlTopQueriesResponse, + GcpInfraTimeseriesResponse, ServiceCloudflareStatsResponse, ServicePlanetScaleStatsResponse, CloudflareInfraZonesResponse, @@ -97,6 +103,7 @@ import { TraceId, SpanId, } from "@maple/domain/http" +import { GCP_INFRA_SERVICE_IDS, gcpInfraMetrics } from "@maple/domain/gcp-infra" import { SESSION_LIVE_WINDOW_SECONDS } from "@maple/domain/query-engine" import { isAiContentFormat } from "@maple/domain/ai-traffic" import { Cause, Clock, DateTime, Effect, Option, Schema } from "effect" @@ -848,6 +855,57 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) + .handle("gcpInfraPresence", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpInfraPresence, tenant, payload) + const reported = new Set(rows.map((row) => row.metric)) + return new GcpInfraPresenceResponse({ + services: GCP_INFRA_SERVICE_IDS.filter((service) => + gcpInfraMetrics(service).some((metric) => reported.has(metric.name)), + ), + }) + }), + ) + .handle("gcpInfraMetrics", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpInfraMetrics, tenant, payload) + return new GcpInfraMetricsResponse({ data: rows }) + }), + ) + .handle("gcpInfraTimeseries", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpInfraTimeseries, tenant, payload) + return new GcpInfraTimeseriesResponse({ + data: rows.map((row) => ({ ...row, bucket: DateTime.formatIso(row.bucket) })), + }) + }), + ) + .handle("gcpInfraSparklines", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpInfraSparklines, tenant, payload) + return new GcpInfraSparklinesResponse({ + data: rows.map((row) => ({ ...row, bucket: DateTime.formatIso(row.bucket) })), + }) + }), + ) + .handle("gcpCloudSqlTopQueries", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpCloudSqlTopQueries, tenant, payload) + return new GcpCloudSqlTopQueriesResponse({ data: rows }) + }), + ) + .handle("gcpCloudSqlDatabases", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.gcpCloudSqlDatabases, tenant, payload) + return new GcpCloudSqlDatabasesResponse({ databases: rows.map((row) => row.database) }) + }), + ) .handle("cloudflareInfraZones", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/api/src/routes/queries.ts b/apps/api/src/routes/queries.ts index 607b4821f7..f020f5357e 100644 --- a/apps/api/src/routes/queries.ts +++ b/apps/api/src/routes/queries.ts @@ -16,6 +16,11 @@ import type { CloudflareInfraZoneTimeseriesRequest, CloudflareInfraZonesRequest, ContainerInfraTimeseriesRequest, + GcpInfraMetricsRequest, + GcpInfraPresenceRequest, + GcpCloudSqlInstanceRequest, + GcpInfraSparklinesRequest, + GcpInfraTimeseriesRequest, GetLogRequest, NodeInfraTimeseriesRequest, PlanetScaleInfraTimeseriesRequest, @@ -289,6 +294,83 @@ const railwayInfraServiceTimeseries = defineQuery({ }), }) +// A small read of the hourly catalog that decides which tabs /infra/gcp shows. +const gcpInfraPresence = defineQuery({ + id: "gcpInfraPresence", + profile: "discovery", + cache: 60, + compile: (payload: GcpInfraPresenceRequest, orgId: OrgId) => + CH.compile(Integrations.gcpInfraPresenceSQL(), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), +}) + +const gcpInfraMetrics = defineQuery({ + id: "gcpInfraMetrics", + profile: "aggregation", + cache: 15, + compile: (payload: GcpInfraMetricsRequest, orgId: OrgId) => + CH.compile(Integrations.gcpInfraMetricsSQL(payload.service), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), +}) + +const gcpInfraTimeseries = defineQuery({ + id: "gcpInfraTimeseries", + profile: "aggregation", + cache: 15, + compile: (payload: GcpInfraTimeseriesRequest, orgId: OrgId) => + CH.compile(Integrations.gcpInfraTimeseriesSQL(payload.source, payload.keys), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + // The poller stores one point per minute. + bucketSeconds: Math.max(60, payload.bucketSeconds), + }), +}) + +const gcpInfraSparklines = defineQuery({ + id: "gcpInfraSparklines", + profile: "aggregation", + cache: 15, + compile: (payload: GcpInfraSparklinesRequest, orgId: OrgId) => + CH.compile(Integrations.gcpInfraSparklinesSQL(payload.service), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + // The poller stores one point per minute. + bucketSeconds: Math.max(60, payload.bucketSeconds), + }), +}) + +const gcpCloudSqlTopQueries = defineQuery({ + id: "gcpCloudSqlTopQueries", + profile: "aggregation", + cache: 15, + compile: (payload: GcpCloudSqlInstanceRequest, orgId: OrgId) => + CH.compile(Integrations.gcpCloudSqlTopQueriesSQL(payload.keys), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), +}) + +const gcpCloudSqlDatabases = defineQuery({ + id: "gcpCloudSqlDatabases", + profile: "list", + cache: 15, + compile: (payload: GcpCloudSqlInstanceRequest, orgId: OrgId) => + CH.compile(Integrations.gcpCloudSqlDatabasesSQL(payload.keys), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), +}) + const zoneDetailParams = (payload: CloudflareInfraZoneDetailRequest, orgId: OrgId) => ({ orgId, serviceName: payload.serviceName, @@ -808,6 +890,12 @@ export const Queries = { planetscaleInfraTimeseries, railwayInfraServices, railwayInfraServiceTimeseries, + gcpInfraPresence, + gcpInfraMetrics, + gcpInfraTimeseries, + gcpInfraSparklines, + gcpCloudSqlTopQueries, + gcpCloudSqlDatabases, // ZoneDetail / PlanetScaleStats sub-queries, declared above. cloudflareInfraZoneDetailStatus, diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index 5e1a5ae350..e7ca5b0e6b 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -213,7 +213,7 @@ describe("v2 gcp integration over HTTP", () => { const path = `${CONNECTORS}/${created.body.id}` const both = await harness.request("POST", `${path}/setup_scripts`, admin.secret, { - log_filter: "include_gke_container_logs", + application_logs: ["cloud_run", "cloud_functions", "app_engine", "gke"], }) expect(both.status).toBe(200) expect(Object.keys(both.body).sort()).toEqual(["cleanup_script", "object", "setup_script"]) @@ -236,13 +236,13 @@ describe("v2 gcp integration over HTTP", () => { // The cleanup script tells Maple that it ran, with the connector's secret. expect(both.body.cleanup_script).toContain("?secret=maple_gcp_") - // Without an option an existing sink keeps its filter and a new one gets the default, which + // Without the list an existing sink keeps its filter and a new one gets the default, which // leaves out GKE container logs; with logs off there is none to carry. const kept = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) expect(kept.body.setup_script).toContain("LOG_FILTER_MODE='keep'") expect(kept.body.setup_script).toContain(` AND NOT resource.type="k8s_container"'\n`) const unknown = await harness.request("POST", `${path}/setup_scripts`, admin.secret, { - log_filter: "everything", + application_logs: ["everything"], }) expect(unknown.status).toBe(400) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.ts b/apps/api/src/routes/v2/integrations-gcp.http.ts index 8255acf78d..dbfb87d6ec 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.ts @@ -103,7 +103,7 @@ export const HttpV2GcpIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "gcpIn const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly) const scripts = yield* gcp.scripts(tenant.orgId, params.id, { - logFilter: payload.log_filter ?? "keep", + applicationLogs: payload.application_logs ?? null, }) return { object: "gcp_connector.setup_scripts" as const, diff --git a/apps/landing/src/content/docs/integrations/gcp-opentelemetry.md b/apps/landing/src/content/docs/integrations/gcp-opentelemetry.md index 85d0e0d903..a9042aeb1e 100644 --- a/apps/landing/src/content/docs/integrations/gcp-opentelemetry.md +++ b/apps/landing/src/content/docs/integrations/gcp-opentelemetry.md @@ -20,7 +20,7 @@ Your workloads send traces, logs and metrics straight to Maple over OpenTelemetr ### GKE 1. Instrument each service with an OpenTelemetry SDK. Export to Maple, or to the [Maple Kubernetes collector](/docs/infrastructure/kubernetes) in the cluster. -2. [Connect Google Cloud](/docs/integrations/gcp#connect) and keep the log filter on **Recommended: no GKE container logs**. +2. [Connect Google Cloud](/docs/integrations/gcp#connect) and leave **Log filter** as it is: no application output is forwarded. The connection then adds node and cluster logs, audit logs and the `gcp.gke.*` metrics. @@ -28,7 +28,7 @@ The connection then adds node and cluster logs, audit logs and the `gcp.gke.*` m 1. Instrument the service with an OpenTelemetry SDK that exports to Maple, with the settings under [Instrument a Cloud Run service](#instrument-a-cloud-run-service). 2. [Connect Google Cloud](/docs/integrations/gcp#connect). -3. If the service exports logs over OpenTelemetry, [leave out its container output](#change-the-filter) too: the recommended filter forwards what a Cloud Run container writes to stdout and stderr. +3. If the service exports logs over OpenTelemetry, untick **Cloud Run** under [**Log filter**](#change-the-filter): by default the connection forwards what a Cloud Run container writes to stdout and stderr. The connection adds Cloud Run's request logs and the `gcp.cloud_run.*` metrics. Cloud Run sends a `traceparent` header with every request. An SDK that continues it gives the server span the trace ID of Google's request log for that request, so the request log, your spans and your log lines join on one trace. @@ -40,35 +40,39 @@ The connection adds Cloud Run's request logs and the `gcp.cloud_run.*` metrics. The connection adds audit logs and the `gcp.gce.*` metrics. -## GKE container logs +## Application output -The recommended filter leaves out `resource.type="k8s_container"`: what the containers of a GKE cluster write to stdout and stderr. +Application output is what the code of a runtime writes to stdout and stderr. The connection forwards none by default: send it over OpenTelemetry. Platform, request and audit logs are forwarded either way. For workloads that can't, click **Can't forward via OpenTelemetry** under **Log filter** and tick their runtimes. -| Reason | Detail | -| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Duplicates | A workload that sends its logs over OpenTelemetry, or a collector that reads pod logs, already delivers each line to Maple. Forwarding it from Google Cloud stores it twice. | -| No trace link | The OpenTelemetry copy carries the trace and span ID of the request that wrote it. The copy from Google Cloud has neither. | -| Volume | Container output grows with traffic. Every forwarded line counts toward your Maple plan and toward the Pub/Sub usage Google bills to your account. | +| Runtime | Left out when unticked | Still forwarded | +| ------------------- | ----------------------------------------------------------------------------------------------------- | ---------------------------------------- | +| **Cloud Run** | `run.googleapis.com/stdout` and `stderr` of services, jobs, worker pools and 2nd gen functions | Request logs, system logs, audit logs | +| **Cloud Functions** | `cloudfunctions.googleapis.com/cloud-functions` of 1st gen functions, with its start and finish lines | Audit logs | +| **App Engine** | `stdout` and `stderr` of `gae_app` | Request logs, audit logs | +| **GKE containers** | Everything of `resource.type="k8s_container"` | Node, pod and cluster events, audit logs | -Include GKE container logs when the workloads don't send their logs over OpenTelemetry and no collector reads them. Cloud Logging is then the only place those lines exist. +Don't tick a runtime whose workloads already send their logs to Maple: -## Change the filter +| Reason | Detail | +| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Duplicates | A workload that exports logs over OpenTelemetry and also prints them, or a collector that reads pod logs, already delivers each line. Forwarding it from Google Cloud stores it twice. | +| No trace link | The OpenTelemetry copy carries the trace and span ID of the request that wrote it. The copy from Google Cloud has neither. | +| Volume | Application output grows with traffic. Every forwarded line counts toward your Maple plan and toward the Pub/Sub usage Google bills to your account. | + +Maple does not check which workloads send their logs another way. A runtime with both kinds of workloads is one tick: tick it when the uninstrumented ones matter more, since Cloud Logging is the only place their lines exist. -On the connection's card, click **Configure** and choose a filter under **Log filter**. Click **Continue to the script**, then copy the script and run it in Cloud Shell again. +Other places your code runs have no tick and are always forwarded: Compute Engine logs collected by the Ops Agent, and the job output of Batch, Dataflow, Dataproc, Cloud Composer and Vertex AI. Leave them out with a filter of your own. -| Log filter | The sink forwards | -| -------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| **Recommended: no GKE container logs** | Platform logs. Leaves out GKE container logs and the [other exclusions](/docs/integrations/gcp#logs). | -| **Include GKE container logs** | The same, plus GKE container logs. Maple asks you to acknowledge the duplicates before you continue. | -| **Keep the sink's current filter** | What it forwards now. Offered, and preselected, once a run has set log forwarding up. | +## Change the filter -For any other filter, paste the script into an editor, edit `LOG_FILTER` near its top, set `LOG_FILTER_MODE` to `set` and run it. To also leave out Cloud Run container output, for services that send their logs over OpenTelemetry, add this at the end of `LOG_FILTER`, inside the quotes: +On the connection's row, click **Configure**. Under **Log filter**, choose **Replace filter**. To forward application output, click **Can't forward via OpenTelemetry** and tick its runtimes. Click **Continue to the script**, then copy the script and run it in Cloud Shell again. -```text -AND NOT log_id("run.googleapis.com/stdout") AND NOT log_id("run.googleapis.com/stderr") -``` +| Log filter | The sink forwards | +| ----------------------- | ------------------------------------------------------------------------------------- | +| **Keep current filter** | What it forwards now. Preselected once a run has set log forwarding up. | +| **Replace filter** | Platform, request and audit logs, plus the application output of any ticked runtimes. | -Cloud Run's request logs are in `run.googleapis.com/requests` and still pass. The filter uses the [Logging query language](https://cloud.google.com/logging/docs/view/logging-query-language). +For any other filter, paste the script into an editor, edit `LOG_FILTER` near its top, set `LOG_FILTER_MODE` to `set` and run it. The filter uses the [Logging query language](https://cloud.google.com/logging/docs/view/logging-query-language). ## Instrument a Cloud Run service diff --git a/apps/landing/src/content/docs/integrations/gcp.md b/apps/landing/src/content/docs/integrations/gcp.md index 0a39beb6b7..194a0d8211 100644 --- a/apps/landing/src/content/docs/integrations/gcp.md +++ b/apps/landing/src/content/docs/integrations/gcp.md @@ -46,13 +46,13 @@ The host project holds Maple's Pub/Sub topic, subscription and read-only service 1. Open **Integrations → Google Cloud** in Maple and click **Connect Google Cloud**. When the page already has a connection, click **Add connection** above the list. The dialog **Connect Google Cloud** opens on its first stage, **Connect**. 2. Under **What to connect**, choose **Organization**, **Folder** or **Project** and enter its ID. An organization or folder covers every project in it, including new ones. 3. For an organization or folder, enter the **Host project ID**. -4. Under **What to collect**, leave **Log forwarding** and **Metrics and resources** ticked, or untick one. To change which logs are forwarded, choose a [log filter](#log-filter) under **Log filter**. +4. Under **Collect**, leave **Log forwarding** and **Metrics and resources** ticked, or untick one. To change which logs are forwarded, choose a [log filter](#log-filter) under **Log filter**. 5. Click **Create and continue**. Maple creates the connection and the dialog moves to its second stage, **Apply in Google Cloud**. Nothing changes in Google Cloud until you run the script. 6. In step 1, click **Open Cloud Shell** and sign in with the roles above. Click **Authorize** if Cloud Shell asks. 7. In step 2, click **Copy script**, paste it into Cloud Shell and press Enter. The script takes about a minute. **Read the script** shows it in the dialog. 8. Return to Maple. Step 3, **Maple is waiting for the run**, updates on its own. Within a minute of the script ending, usually in seconds, the dialog shows **Google Cloud matches this configuration**. Click **Done**. -If you close the dialog before the script has run, the connection's card shows **Setup pending**. Click **Finish setup** to open the dialog at the script. After five minutes without a report from the script, step 3 reads **Maple has no confirmation yet** and lists what to check. +If you close the dialog before the script has run, the connection's row shows **Setup pending**. Click **Finish setup** to open the dialog at the script. After five minutes without a report from the script, step 3 reads **Maple has no confirmation yet** and lists what to check. A first run for a project prints: @@ -117,8 +117,8 @@ A project, folder or organization can be connected once per Maple organization. ### Change what a connection collects -1. On the connection's card, click **Configure**. The dialog opens on its first stage, **Configure**. -2. Under **What to collect**, tick or untick **Log forwarding** or **Metrics and resources**. The line under each checkbox says what saving that choice does. +1. On the connection's row, click **Configure**. The dialog opens on its first stage, **Configure**. +2. Under **Collect**, tick or untick **Log forwarding** or **Metrics and resources**. The line beside a changed checkbox says what saving it does. 3. Click **Save and continue**. For a change that takes effect in Maple at once, Maple asks first, in a dialog titled for example "Turn log forwarding off for Project acme-prod?". The dialog says what happens **In Maple** and what goes on **In Google Cloud**. Click **Turn off and continue**, or **Turn on** when you turn something back on that still exists in Google Cloud. Then the button of the first stage reads **Save**, and there is nothing to run. 4. The dialog moves to **Apply in Google Cloud**. Its recap says what is saved and what this run does. Copy the script and run it in Cloud Shell again. 5. When the run reports, the dialog shows **Google Cloud matches this configuration**. Click **Done**. @@ -132,7 +132,7 @@ A change takes effect in Maple and in Google Cloud at different times: | **Log forwarding** or **Metrics and resources** on | Starts once the script has run. | The script creates the log sink, or the read-only service account. | | Back on while Google Cloud still has the log sink or the service account | At once: Maple stores the logs again as soon as you save, and reads metrics again from the next 5-minute read. | Nothing changes. | -Until the script has run, the connection's card shows **Changes not applied in Google Cloud** and says what goes on until then, for example "Google Cloud keeps forwarding logs, billed by Google, until the script has removed the log sink." Click **Apply changes** to open the dialog at the script. +Until the script has run, the connection's row shows **Changes not applied in Google Cloud** and says what goes on until then, for example "Google Cloud keeps forwarding logs, billed by Google, until the script has removed the log sink." Click **Apply changes** to open the dialog at the script. A run that removes the log sink takes one to two minutes: after it deletes the sink it waits a minute for Google to stop routing to the topic, so your project logs no sink error. @@ -142,54 +142,70 @@ A connection collects at least one of the two: the last one that is on can't be ### Log filter -**Log filter** decides which filter the script writes onto the sink. It is in the first stage of **Connect Google Cloud** and of **Configure**, below **What to collect**, while **Log forwarding** is ticked. To change it on a connection, click **Configure**, choose a filter, click **Continue to the script** and run the script again. +**Log filter** decides which filter the script writes onto the sink. It is in the first stage of **Connect Google Cloud** and of **Configure**, below **Collect**, while **Log forwarding** is ticked. -| Log filter | The script | -| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Recommended: no GKE container logs** | Sets the [recommended filter](#logs). Preselected for a connection that has no sink yet. | -| **Include GKE container logs** | Sets the recommended filter without its GKE container clause. Maple asks you to acknowledge the duplicates before you continue. | -| **Keep the sink's current filter** | Leaves an existing sink's filter as it is. Offered, and preselected, once a run has set log forwarding up. Maple can't see the sink's filter: read it in the console under **Logging → [Log Router](https://console.cloud.google.com/logs/router)**. | +Platform, request and audit logs are always forwarded. Application output, what the code of a runtime writes to stdout and stderr, is not: send it with your [OpenTelemetry setup](/docs/integrations/gcp-opentelemetry), where each line links to its trace. -Include GKE container logs only for workloads that don't send their logs to Maple over OpenTelemetry. Otherwise each line is stored twice: see [GKE container logs](/docs/integrations/gcp-opentelemetry#gke-container-logs). +For workloads that can't send logs over OpenTelemetry, click **Can't forward via OpenTelemetry** and tick the runtimes whose application output to forward. None is ticked until you do. **Use OpenTelemetry instead** closes the ticks and forwards none again. -Choosing **Include GKE container logs** shows the notice **Not recommended when your GKE workloads send traces to Maple**. The button that continues stays disabled until you tick **I understand that GKE container logs can be stored twice**. To back out, click **Use recommended filter**, or **Keep current filter** on a connection that has a sink. +| Runtime | Covers | +| ------------------- | --------------------------------------------------- | +| **Cloud Run** | Services, jobs, worker pools and 2nd gen functions. | +| **Cloud Functions** | 1st gen functions. | +| **App Engine** | Standard and flexible. | +| **GKE containers** | Every container of a cluster. | -For any other filter, paste the script into an editor first, edit `LOG_FILTER` near its top and set `LOG_FILTER_MODE` to `set`. The filter uses the [Logging query language](https://cloud.google.com/logging/docs/view/logging-query-language). The script checks the filter with Google before it creates anything. Later runs with **Keep the sink's current filter** leave your filter in place, while the other two options replace it. +Don't tick a runtime whose workloads already send their logs to Maple over OpenTelemetry. Otherwise each line is stored twice: see [Application output](/docs/integrations/gcp-opentelemetry#application-output). Maple does not check which workloads do. + +On a connection that already has a sink, the choice starts on **Keep current filter**: the script leaves the sink's filter as it is. Maple can't see that filter, so **Current filter in Log Router** opens it in the console. To change it, click **Configure**, choose **Replace filter**, set the runtimes, click **Continue to the script** and run the script again. + +For any other filter, paste the script into an editor first, edit `LOG_FILTER` near its top and set `LOG_FILTER_MODE` to `set`. The filter uses the [Logging query language](https://cloud.google.com/logging/docs/view/logging-query-language). The script checks the filter with Google before it creates anything. Later runs with **Keep current filter** leave your filter in place, while **Replace filter** replaces it. ## Verify -Under **Connections**, each connection has a card, the worst state first. A card shows a status for **Log forwarding** and one for **Metrics and resources**. The page updates on its own. - -| Log forwarding | Meaning | -| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Not set up** | The setup script has not set log forwarding up yet. Run it. | -| **Setup running** | A run reported on metrics and resources in the last two minutes and has not reported on log forwarding yet. | -| **Waiting for first logs** | The script ran and no entry has arrived. After 20 minutes the row shows the hint **No log entry in 20 minutes** with a `gcloud logging write` command that writes a test entry. | -| **Receiving logs** | Entries arrive. The row shows when the last one did, and **View logs** opens Logs filtered to the connection. | -| **No logs in 24 hours** | Nothing passed the filter for a day. Not an error. | -| **Rejecting logs** | Maple refused the last push. The row says why and what to do. | -| **Off** | Turned off in **Configure**. While Google Cloud still has the log sink, the row reads "Maple discards what Google Cloud still forwards". | - -| Metrics and resources | Meaning | -| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Not set up** | The setup script has not granted Maple read access yet. Run it. | -| **Setup running** | A run reported on log forwarding in the last two minutes and has not reported on metrics and resources yet. | -| **Waiting for first metrics** | The script ran and the first read has not landed. After 15 minutes the row says that no metrics have arrived yet. | -| **Receiving metrics** | Reads succeed. The row shows when the last metrics arrived, and **Open Infrastructure** opens **Infrastructure → Google Cloud**. A folder or organization also shows how many projects Maple found. The label reads **Receiving metrics, resource list incomplete** when Maple could not list every resource. | -| **Receiving metrics, incomplete** | The last read is under 10 minutes old, but some metric queries failed or were cut short. The row says which. | -| **Metrics stalled** | No read for 30 minutes and no error. Maple retries every 5 minutes. | -| **Can't read metrics** | Reads fail, and none has succeeded in the last 10 minutes. The row says why and what to do. | -| **Off** | Turned off in **Configure**. While the read-only service account still exists, the row reads "Maple no longer uses its service account". | +Once a connection has delivered data, **Integrations → Google Cloud** opens with a row of numbers, summed over every connection: + +| Number | Counts | +| --------------------- | ----------------------------------------------------------------------- | +| **Log entries · 24h** | Entries forwarded in the last 24 hours, with a bar per hour. | +| **Errors · 24h** | The entries among them of severity error and above. | +| **Workloads · 24h** | Workloads that reported metrics in the last 24 hours. | +| **Resources** | Resources in the [inventory](#resources), and the projects they are in. | + +The row follows what has delivered. While only **Log forwarding** delivers, it shows **Log entries**, **Errors** and **Warnings**. While only **Metrics and resources** delivers, it shows **Workloads**, **Resources** and **Projects**. A number Maple cannot count yet shows a dash. Below the numbers, **Workloads** lists what reported metrics in the last 24 hours, the worst first, each a link to its page under **Infrastructure → Google Cloud**. Beside it, **Resources** counts the [inventory](#resources) by type. + +Under **Connections**, each connection is a row, the worst state first, with a status for **Logs** and one for **Metrics**. A hint, a warning or a failure shows in a band under the row. The page updates on its own. + +| Logs | Meaning | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Not set up** | The setup script has not set log forwarding up yet. Run it. | +| **Setup running** | A run reported on metrics and resources in the last two minutes and has not reported on log forwarding yet. | +| **Waiting for the first entry** | The script ran and no entry has arrived. After 20 minutes the band reads **No log entry in 20 minutes** with a `gcloud logging write` command that writes a test entry. | +| **Receiving** | Entries arrive. The row shows when the last one did. | +| **Nothing in 24 hours** | Nothing passed the filter for a day. Not an error. | +| **Rejected** | Maple refused the last push. The band says why and what to do. | +| **Off** | Turned off in **Configure**. While Google Cloud still has the log sink, the band **Changes not applied in Google Cloud** says so. | + +| Metrics | Meaning | +| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Not set up** | The setup script has not granted Maple read access yet. Run it. | +| **Setup running** | A run reported on log forwarding in the last two minutes and has not reported on metrics and resources yet. | +| **Waiting for the first read** | The script ran and the first read has not landed. After 15 minutes the band says that no metrics have arrived yet. | +| **Receiving** | Reads succeed. The row shows when the last one landed. The label reads **Receiving, resource list incomplete** when Maple could not list every resource. | +| **Incomplete** | The last read is under 10 minutes old, but some metric queries failed or were cut short. The band says which. | +| **Stalled** | No read for 30 minutes and no error. Maple retries every 5 minutes. | +| **Can't be read** | Reads fail, and none has succeeded in the last 10 minutes. The band says why and what to do. | +| **Off** | Turned off in **Configure**. While the read-only service account still exists, the band **Changes not applied in Google Cloud** says so. | The page header and the Integrations list show the connection's worst status: **Needs attention**, **Setup pending**, **Changes pending**, **Waiting for data** or **Healthy**. -A card whose connection needs the setup script shows a band with one button. The button opens **Configure** at its second stage, **Apply in Google Cloud**. +A connection that needs the setup script shows a band with one button. The button opens **Configure** at its second stage, **Apply in Google Cloud**. | Band | Button | Shown when | | --------------------------------------- | ------------------------ | ---------------------------------------------------------------------------------------------------------------------- | | **Setup pending** | **Finish setup** | The script of a new connection has not reported, or its last run stopped part of the way. | | **Changes not applied in Google Cloud** | **Apply changes** | The saved configuration and what the script last reported differ. The band says what goes on until the script has run. | -| **The setup script needs to run again** | **Run the script again** | The text of a failure asks for another run of the setup script. | +| A failure's own message | **Run the script again** | The text of the failure asks for another run of the setup script. | Members who are not admins of the Maple organization see the statuses and the numbers, without **Configure**, the **⋮** menu or the band's button. The band asks them to have an admin run the setup script, and a line under the list says "Only Maple organization admins can add, configure or disconnect connections." @@ -198,6 +214,7 @@ Then check the data: 1. Open [Logs](/docs/explore/logs) and look for a service named after one of your Cloud Run services, functions or instances. 2. In the [metrics explorer](/docs/explore/metrics), search for the prefix `gcp`. 3. Under **Dashboards → Browse templates**, select **Google Cloud** and click **Create dashboard**. It has one tab per service. Use the **Project**, **Region** and **Workload** filters in the toolbar to narrow every chart. To open it on one project, first fill in **Project ID** under **Parameters**, below the preview. +4. Open **Infrastructure → Google Cloud** for a table of your workloads per service. ## Permissions @@ -239,15 +256,15 @@ Before it changes anything, the script asks Google which permissions the signed- ### Logs -Maple receives every entry the sink's filter lets through. The recommended filter excludes: +Maple receives every entry the sink's filter lets through. The default filter excludes: -| Excluded | Reason | -| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Data Access audit logs | They record every API read. | -| Load balancer health checks | Probes hit each backend every few seconds. | -| Kubernetes lease renewals | A GKE cluster renews its leader-election leases all day: hundreds of audit entries a minute for an idle node. | -| VM serial console output | A VM writes thousands of lines of raw terminal output at boot. | -| GKE container logs | Workloads that send logs over OpenTelemetry already deliver them to Maple, so each line would be stored twice. See [GKE container logs](/docs/integrations/gcp-opentelemetry#gke-container-logs). | +| Excluded | Reason | +| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Data Access audit logs | They record every API read. | +| Load balancer health checks | Probes hit each backend every few seconds. | +| Kubernetes lease renewals | A GKE cluster renews its leader-election leases all day: hundreds of audit entries a minute for an idle node. | +| VM serial console output | A VM writes thousands of lines of raw terminal output at boot. | +| Application output of unticked runtimes | By default every runtime: workloads send their logs over OpenTelemetry, so each line would be stored twice. See [Application output](/docs/integrations/gcp-opentelemetry#application-output). | The filter, one clause per line in the same order: @@ -256,10 +273,13 @@ NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update" AND NOT logName:"serialconsole.googleapis.com" +AND NOT (log_id("run.googleapis.com/stdout") OR log_id("run.googleapis.com/stderr")) +AND NOT log_id("cloudfunctions.googleapis.com/cloud-functions") +AND NOT (resource.type="gae_app" AND (log_id("stdout") OR log_id("stderr"))) AND NOT resource.type="k8s_container" ``` -**Include GKE container logs** sets the filter without the last clause. To forward different logs, see [Log filter](#log-filter). +The last four clauses are the application output of each runtime: ticking one under [Log filter](#log-filter) removes its clause. The script reports to Maple through a log named `maple-setup`. Maple reads that entry as the report and does not store it, so don't write your own logs under that name. @@ -386,6 +406,68 @@ Every series also carries `cloud.account.id` (the project ID) and, for resources Every hour Maple lists these resources from Cloud Asset Inventory, with their project, location, state and labels: projects, Cloud Run services and jobs, Cloud Functions, GKE clusters, Compute Engine instances, Cloud SQL instances, Pub/Sub topics and subscriptions, and load balancer URL maps, backend services and forwarding rules. +## Infrastructure → Google Cloud + +**Infrastructure → Google Cloud** is in the sidebar and on the Infrastructure overview while a connection has **Metrics and resources** on. + +The page opens with a band that counts the workloads of every service by health, then one tab per service that reported metrics in the selected time range, with one row per workload. The tab you are on stays when you change the range. Counts and byte totals cover the time range; everything else is the average over it. + +| Band cell | Workloads it counts | +| --------- | --------------------------------------------------------------------------------------- | +| Saturated | Using 90% or more of a CPU, memory or disk limit, on average over the time range | +| Elevated | Using 60% or more of one | +| Erroring | Failing 1% or more of at least 100 requests, executions or push deliveries in the range | + +Click a cell to narrow every service tab to those workloads, and click it again to clear it. While a cell is active each tab shows how many of its workloads it holds. + +| Tab | One row per | Columns | +| --------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| Cloud Run | Service, project and region | Requests, 5xx rate, latency p95 and p99, active instances, CPU p95, memory p95 | +| Cloud Functions | 1st gen function, project and region | Executions, error rate, duration p95 and p99, active instances, memory p95 | +| GKE | Container name, namespace and cluster | CPU cores, CPU of limit, memory, memory of limit, restarts | +| Compute Engine | Instance, project and zone | CPU, memory (E2 machine types only), network in and out, disk read and write | +| Cloud SQL | Instance, project and region | CPU, memory, disk, connections, replica lag | +| Pub/Sub | Subscription and project | Backlog, age of the oldest unacknowledged message, delivered, acknowledged, dead-lettered, push errors | +| Load Balancing | URL map, backend and project | Requests, 5xx rate, latency p95 and p99, backend latency p95, bytes sent | +| Resources | Resource from the [inventory](#resources) | Type, project, location, state and labels, with a filter by type and by project | + +Search a tab by name, project or location, filter every service tab by project and region, and click a column to sort by it. The search, the filters and the band cell are part of the page's address, so a link opens the same view. A dash means the workload did not report that metric. CPU below one core is written in millicores: `250m` is a quarter of a core. Cloud Functions (2nd gen) run on Cloud Run and appear on the Cloud Run tab. Pub/Sub topic metrics are collected but have no tab: chart them in a dashboard. + +### Workload pages + +Click a row to open that workload: its headline numbers, each marked as a total or an average over the time range, then its charts. + +| Service | Charts | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cloud Run | Requests by response class, 5xx rate, request latency, active and idle instances, CPU and memory utilization, peak concurrent requests per instance, billable instances | +| Cloud Functions | Executions by status, error rate, execution time, active and idle instances, memory per execution, network egress | +| GKE | CPU cores, CPU of limit, memory, memory of limit and restarts of the container, then CPU and memory of allocatable for the nodes of its cluster | +| Compute Engine | CPU utilization, memory used (E2 machine types only), network received and sent, disk read and written | +| Cloud SQL | CPU, memory and disk utilization, connections, disk read and write operations, replica lag (read replicas only) | +| Pub/Sub | Backlog, age of the oldest unacknowledged message, delivered and acknowledged messages, dead-lettered messages, push requests by response and push error rate (push only) | +| Load Balancing | Requests by response class, 5xx rate, total latency, backend latency, backend requests by response class, request and response traffic | + +Latency charts draw p50, p95 and p99, and so do Cloud Run's CPU, memory and concurrency and a function's memory; every other line is the average. Requests and executions by class, traffic, messages and disk operations are per second; restarts and dead-lettered messages are counts per point. A point covers at least 5 minutes, and more on a long range. While a workload reports nothing, as a Cloud Run service scaled to zero does, its counters read zero and its other lines have a gap. A share of a limit carries a line at 80%. Hovering one chart marks the same moment on the others. + +From a workload's page: + +- **Logs** opens the workload's logs. A Cloud SQL instance, a subscription and a URL map open the logs of their resource type, narrowed to the resource. +- **Traces** is there when a service of the same name sent traces in the time range. +- The icon beside a chart's title opens its metric in the metrics explorer. +- When the workload is in the [inventory](#resources), its state and labels show under its name, and **Google Cloud** opens its product's list in the Google Cloud console, on the resource's project. A GKE container gets the link to its cluster's list, without state or labels. The match is by type, project, name and location: a workload Maple cannot match exactly shows neither. + +### Elsewhere in Maple + +- The **Infrastructure overview** row counts the workloads, colors them by health and lists up to three that are saturated, elevated or erroring. Each opens the workload's page. While every workload is healthy the row names the services that report. +- On the **service map**, the panel of a traced service shows the numbers of the Cloud Run service, function, GKE container or VM of the same name, with a link to its page. +- A row of the **Resources** tab that is one workload (a Cloud Run service, a 1st gen function, a VM, a Cloud SQL instance, a subscription) opens that workload's page. + +The **Resources** tab shows the first 500 resources that match its filters, and how many projects Maple found. It does not depend on the time range. + +Until a connection's setup script has run, the page reads **Finish setting up Google Cloud**. After the run it reads **Collecting your first Google Cloud metrics** until the first read lands, within about 10 minutes. + +With **Metrics and resources** off on every connection, the page leaves the sidebar and the overview. Its address still opens it: for a time range with data it shows the service tabs with what Maple collected before, under the notice **Google Cloud metrics are switched off**, and otherwise it reads **Turn on metrics for Google Cloud**. The **Resources** tab is back once a connection collects again. + ## Google Cloud costs Google bills Pub/Sub and Cloud Monitoring usage to your account, separately from your Maple plan. @@ -409,7 +491,7 @@ Cloud Asset Inventory searches are free of charge. See [Cloud Asset Inventory pr ## Troubleshooting -To run the setup script again, click **Configure** on the connection's card, then **Continue to the script**. When a failure asks for another run, the card also shows **The setup script needs to run again** with the button **Run the script again**. +To run the setup script again, click **Configure** on the connection's row, then **Continue to the script**. When a failure asks for another run, its band has the button **Run the script again**. ### In Cloud Shell @@ -434,23 +516,23 @@ When you create a connection in Maple, "The Google Cloud project acme-prod is al ### Log forwarding -| The connection shows | Cause and fix | -| ------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Setup pending** after the script ran | The script stopped before it finished, or could not reach Maple. Read its last lines in Cloud Shell and paste it again. | -| **Waiting for first logs** for more than 20 minutes | Either nothing was logged that passes the filter, or the sink can't publish. Run the `gcloud logging write` command the row shows. If the entry does not arrive within a minute, open **Configure** and run the setup script again. | -| **Rejecting logs**: "The Pub/Sub subscription wraps each entry in an envelope Maple can't read." | The subscription was changed to deliver wrapped messages. Run the setup script again: it resets the subscription. Entries sent meanwhile are lost. | -| **Rejecting logs**: "Maple could not store an entry just now." | Nothing to do. Pub/Sub retries the entry for up to a day, and the status returns to **Receiving logs** with the next accepted entry. | -| **Rejecting logs**: "This Maple organization is over its plan limit, so Maple refuses new logs." | Raise the plan limit under **Settings → Billing**. Pub/Sub retries refused entries for up to a day. | -| GKE container logs appear twice | The sink forwards GKE container logs and the workloads also send them over OpenTelemetry. In **Configure**, choose **Recommended: no GKE container logs** as the [log filter](#log-filter) and run the script again. | -| GKE container logs are missing | The recommended filter leaves them out. If the workloads don't send their logs over OpenTelemetry, choose **Include GKE container logs** as the [log filter](#log-filter) in **Configure** and run the script again. | +| The connection shows | Cause and fix | +| ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Setup pending** after the script ran | The script stopped before it finished, or could not reach Maple. Read its last lines in Cloud Shell and paste it again. | +| **Waiting for the first entry** for more than 20 minutes | Either nothing was logged that passes the filter, or the sink can't publish. Run the `gcloud logging write` command the band shows. If the entry does not arrive within a minute, open **Configure** and run the setup script again. | +| **Rejected**: "The Pub/Sub subscription wraps each entry in an envelope Maple can't read." | The subscription was changed to deliver wrapped messages. Run the setup script again: it resets the subscription. Entries sent meanwhile are lost. | +| **Rejected**: "Maple could not store an entry just now." | Nothing to do. Pub/Sub retries the entry for up to a day, and the status returns to **Receiving** with the next accepted entry. | +| **Rejected**: "This Maple organization is over its plan limit, so Maple refuses new logs." | Raise the plan limit under **Settings → Billing**. Pub/Sub retries refused entries for up to a day. | +| A workload's logs appear twice | The sink forwards its application output and the workload also sends its logs over OpenTelemetry. In **Configure**, choose **Replace filter**, leave its runtime unticked under [Log filter](#log-filter) and run the script again. | +| A workload's stdout and stderr are missing | Application output is not forwarded by default. Send it over OpenTelemetry, or tick its runtime under [Log filter](#log-filter) in **Configure** and run the script again. | ### Metrics and resources -**Can't read metrics** and **Receiving metrics, incomplete** come with one of the messages below. The first means no read has succeeded in the last 10 minutes. The second means the last read is under 10 minutes old and something is missing. The messages about the resource list show under a green **Receiving metrics**. Where a project's message says "the project", a folder's or organization's says "the host project". +**Can't be read** and **Incomplete** come with one of the messages below. The first means no read has succeeded in the last 10 minutes. The second means the last read is under 10 minutes old and something is missing. The messages about the resource list show under **Receiving, resource list incomplete**. Where a project's message says "the project", a folder's or organization's says "the host project". | The connection shows | Cause and fix | | ----------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Metrics stalled** | No read for 30 minutes and no error. Maple retries on its own. If it stays for more than an hour, write to support. | +| **Stalled** | No read for 30 minutes and no error. Maple retries on its own. If it stays for more than an hour, write to support. | | "Maple can't sign in as this connection's read-only service account yet." | Shown from 10 minutes after a setup run, on a connection Maple has never read. The account does not exist, or the grant to Maple is missing or blocked by an organization policy. Run the setup script again and read its last lines. See [Domain restricted sharing](#domain-restricted-sharing). | | "Maple can no longer sign in as this connection's read-only service account." | Maple has read this connection before. The account was deleted, or the grant to Maple was removed or is blocked by an organization policy. Run the setup script again: it restores both, or says what blocks it. | | "The project acme-prod has no active billing account" | Link a billing account to the host project. The message carries the link. Maple retries every 5 minutes. | @@ -484,7 +566,7 @@ Google checks the policy when a grant is made, so leave the exception in place: ## Disconnect -1. On **Integrations → Google Cloud**, open the **⋮** menu (**More actions**) on the connection's card and choose **Disconnect**. +1. On **Integrations → Google Cloud**, open the **⋮** menu (**More actions**) on the connection's row and choose **Disconnect**. 2. Click **Copy cleanup script** and run it in Cloud Shell. It deletes the log sink, topic, subscription and read-only service account, and ends with "Done. Everything the setup script created is gone." 3. Wait for the check mark in the dialog, then click **Disconnect**. Maple stops reading the connection's metrics, and stops accepting its logs within about a minute. Data already in Maple is kept. diff --git a/apps/web/perf/check-bundle-budget.ts b/apps/web/perf/check-bundle-budget.ts index 04f58a87cb..92a7d9ccf3 100644 --- a/apps/web/perf/check-bundle-budget.ts +++ b/apps/web/perf/check-bundle-budget.ts @@ -108,8 +108,18 @@ const gzipBytes = chunks.reduce((total, chunk) => total + chunk.gzipBytes, 0) // 699.5 KB from the Google Cloud integration card (#1334, 2026-10-15): ~0.9 KB // of startup measured in CI against #1333's 697.3 from the card and its // contract additions. -// 700.5 KB with main's 1 KB from #1494, #1500, #1502 merged up the stack. -const maxGzipBytes = 700.5 * 1024 +// 701.5 KB from the Infrastructure → Google Cloud page (#1336, 2026-10-15): +// ~2 KB of startup measured in CI against #1335's 698.3 — the inventory +// schemas on the contract, the connector status rules the sidebar and the +// integrations hub read, and the Infrastructure nav entry with its icon. The +// /infra/gcp pages stay in their route chunks. +// 705.5 KB from the Google Cloud page redesign and Cloud SQL data (#1336, 2026-10-16): +// ~3.8 KB of startup measured in CI on the stack's top at 708.2 against #1480's 704.4 — the +// sparkline, top-statement and database endpoints on the query-engine contract, the Cloud SQL +// settings on the inventory schema, and two lab registry entries. The pages stay in their +// route chunks. +// 706.5 KB with main's 1 KB from #1494, #1500, #1502 merged up the stack. +const maxGzipBytes = 706.5 * 1024 const budgetLabel = `${(maxGzipBytes / 1024).toFixed(1)} KB` // Anything lazy-only: chat, replay, and every dev-only lab surface. The diff --git a/apps/web/src/api/warehouse/gcp-infra.ts b/apps/web/src/api/warehouse/gcp-infra.ts new file mode 100644 index 0000000000..5bcbc69537 --- /dev/null +++ b/apps/web/src/api/warehouse/gcp-infra.ts @@ -0,0 +1,253 @@ +import { Effect, Schema } from "effect" +import { GCP_INFRA_SERVICE_IDS, GCP_INFRA_SOURCE_IDS } from "@maple/domain/gcp-infra" +import { + GcpCloudSqlInstanceRequest, + GcpInfraMetricsRequest, + GcpInfraPresenceRequest, + GcpInfraSparklinesRequest, + GcpInfraTimeseriesRequest, +} from "@maple/domain/http" +import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" +import { WarehouseDateTimeString, decodeInput, runWarehouseQuery } from "@/api/warehouse/effect-utils" +import { getServiceDbQuerySummary } from "@/api/warehouse/service-map" + +/** /infra/gcp data access over the `gcp.*` metrics the Google Cloud poller stores. */ + +const WindowInputSchema = Schema.Struct({ + startTime: WarehouseDateTimeString, + endTime: WarehouseDateTimeString, +}) + +const MetricsInputSchema = Schema.Struct({ + ...WindowInputSchema.fields, + service: Schema.Literals(GCP_INFRA_SERVICE_IDS), +}) + +const TimeseriesInputSchema = Schema.Struct({ + ...WindowInputSchema.fields, + bucketSeconds: Schema.Number, + source: Schema.Literals(GCP_INFRA_SOURCE_IDS), + keys: Schema.Array(Schema.String), +}) + +const SparklinesInputSchema = Schema.Struct({ + ...WindowInputSchema.fields, + bucketSeconds: Schema.Number, + services: Schema.Array(Schema.Literals(GCP_INFRA_SERVICE_IDS)), +}) + +const CloudSqlInputSchema = Schema.Struct({ + ...WindowInputSchema.fields, + /** The instance's identity values, as `TimeseriesInputSchema`'s. */ + keys: Schema.Array(Schema.String), +}) + +/** The services with a metric in the window. */ +const getGcpInfraPresence = Effect.fn("QueryEngine.getGcpInfraPresence")(function* ({ + data, +}: { + data: (typeof WindowInputSchema)["Encoded"] +}) { + const input = yield* decodeInput(WindowInputSchema, data, "getGcpInfraPresence") + const result = yield* runWarehouseQuery("gcpInfraPresence", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpInfraPresence({ + payload: new GcpInfraPresenceRequest(input), + }) + }), + ) + return { services: result.services } +}) + +/** One service's metrics: a point per workload, metric and label value. */ +const getGcpInfraMetrics = Effect.fn("QueryEngine.getGcpInfraMetrics")(function* ({ + data, +}: { + data: (typeof MetricsInputSchema)["Encoded"] +}) { + const input = yield* decodeInput(MetricsInputSchema, data, "getGcpInfraMetrics") + const result = yield* runWarehouseQuery("gcpInfraMetrics", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpInfraMetrics({ + payload: new GcpInfraMetricsRequest(input), + }) + }), + ) + return { points: result.data } +}) + +/** + * Every reporting service's metrics at once: what the summary band, the tabs and the + * Infrastructure overview read. One presence probe, then one query per reporting service; a + * service whose query fails is returned empty and `failed`, so the others still show. + */ +export const getGcpInfraFleet = Effect.fn("QueryEngine.getGcpInfraFleet")(function* ({ + data, +}: { + data: (typeof WindowInputSchema)["Encoded"] +}) { + const { services } = yield* getGcpInfraPresence({ data }) + return { + services: yield* Effect.forEach( + services, + (service) => + getGcpInfraMetrics({ data: { ...data, service } }).pipe( + Effect.map(({ points }) => ({ service, points, failed: false })), + Effect.orElseSucceed(() => ({ service, points: [], failed: true })), + ), + { concurrency: "unbounded" }, + ), + } +}) + +/** One workload's metrics over time: a point per bucket, metric and label value. */ +export const getGcpInfraTimeseries = Effect.fn("QueryEngine.getGcpInfraTimeseries")(function* ({ + data, +}: { + data: (typeof TimeseriesInputSchema)["Encoded"] +}) { + const input = yield* decodeInput(TimeseriesInputSchema, data, "getGcpInfraTimeseries") + const result = yield* runWarehouseQuery("gcpInfraTimeseries", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpInfraTimeseries({ + payload: new GcpInfraTimeseriesRequest(input), + }) + }), + ) + return { points: result.data } +}) + +/** + * The row sparklines of one list view, one query per service. A service whose query fails is + * returned empty: its rows draw no sparkline, and the list still shows. + */ +export const getGcpInfraSparklines = Effect.fn("QueryEngine.getGcpInfraSparklines")(function* ({ + data, +}: { + data: (typeof SparklinesInputSchema)["Encoded"] +}) { + const { services, ...input } = yield* decodeInput(SparklinesInputSchema, data, "getGcpInfraSparklines") + return { + services: yield* Effect.forEach( + services, + (service) => + runWarehouseQuery("gcpInfraSparklines", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpInfraSparklines({ + payload: new GcpInfraSparklinesRequest({ ...input, service }), + }) + }), + ).pipe( + Effect.map((result) => ({ service, points: result.data })), + Effect.orElseSucceed(() => ({ service, points: [] })), + ), + { concurrency: "unbounded" }, + ), + } +}) + +/** A Cloud SQL instance's statements by total execution time, as the poller stored Query Insights'. */ +export const getGcpCloudSqlTopQueries = Effect.fn("QueryEngine.getGcpCloudSqlTopQueries")(function* ({ + data, +}: { + data: (typeof CloudSqlInputSchema)["Encoded"] +}) { + const input = yield* decodeInput(CloudSqlInputSchema, data, "getGcpCloudSqlTopQueries") + const result = yield* runWarehouseQuery("gcpCloudSqlTopQueries", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpCloudSqlTopQueries({ + payload: new GcpCloudSqlInstanceRequest(input), + }) + }), + ) + return { statements: result.data } +}) + +const DatabaseInstancesInputSchema = Schema.Struct({ + ...WindowInputSchema.fields, + /** A `db.namespace` the traces query. */ + database: Schema.String, + /** The identity values of each Cloud SQL instance to look in. */ + instances: Schema.Array(Schema.Array(Schema.String)), +}) + +/** + * The PostgreSQL instances that hold a database of the traces' name, from the connections the + * poller stored per database. An instance whose read fails is left out. + */ +export const getGcpCloudSqlInstancesOfDatabase = Effect.fn("QueryEngine.getGcpCloudSqlInstancesOfDatabase")( + function* ({ data }: { data: (typeof DatabaseInstancesInputSchema)["Encoded"] }) { + const { database, instances, ...window } = yield* decodeInput( + DatabaseInstancesInputSchema, + data, + "getGcpCloudSqlInstancesOfDatabase", + ) + const owners = yield* Effect.forEach( + instances, + (keys) => + runWarehouseQuery("gcpCloudSqlDatabases", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpCloudSqlDatabases({ + payload: new GcpCloudSqlInstanceRequest({ ...window, keys }), + }) + }), + ).pipe( + Effect.map((result) => (result.databases.includes(database) ? [keys] : [])), + Effect.orElseSucceed(() => []), + ), + { concurrency: 4 }, + ) + return { instances: owners.flat() } + }, +) + +/** Cloud SQL's own databases, and PostgreSQL's templates: no application queries them. */ +const SYSTEM_DATABASES = new Set(["cloudsqladmin", "template0", "template1", "postgres"]) + +/** + * A PostgreSQL instance as the traces see it: its databases, from the connections the poller + * stored per database in the window, and the query shapes the traces recorded against each one + * (`db.namespace`). `services` is the most services any one database had calling it. + */ +export const getGcpCloudSqlTraceSide = Effect.fn("QueryEngine.getGcpCloudSqlTraceSide")(function* ({ + data, +}: { + data: (typeof CloudSqlInputSchema)["Encoded"] +}) { + const input = yield* decodeInput(CloudSqlInputSchema, data, "getGcpCloudSqlTraceSide") + const result = yield* runWarehouseQuery("gcpCloudSqlDatabases", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.gcpCloudSqlDatabases({ + payload: new GcpCloudSqlInstanceRequest(input), + }) + }), + ) + const databases = result.databases.filter((database) => !SYSTEM_DATABASES.has(database)) + // A database whose read fails is left out; the others still show. + const summaries = (yield* Effect.forEach( + databases, + (database) => + getServiceDbQuerySummary({ + data: { + dbSystem: "postgresql", + dbNamespace: database, + startTime: input.startTime, + endTime: input.endTime, + topN: 50, + }, + }).pipe(Effect.orElseSucceed(() => null)), + { concurrency: 4 }, + )).filter((summary) => summary !== null) + return { + databases, + tracedQueries: summaries.flatMap((summary) => summary.topQueries), + services: Math.max(0, ...summaries.map((summary) => summary.summary?.activeServiceCount ?? 0)), + } +}) diff --git a/apps/web/src/components/dashboard/app-sidebar.tsx b/apps/web/src/components/dashboard/app-sidebar.tsx index 0110b525f7..48417f1991 100644 --- a/apps/web/src/components/dashboard/app-sidebar.tsx +++ b/apps/web/src/components/dashboard/app-sidebar.tsx @@ -302,7 +302,8 @@ function NavRow({ const seen = new Set() const unique: NavSubItem[] = [] for (const sub of item.subItems) { - if (!sub.icon || seen.has(sub.icon)) continue + // The section's own front door (Infrastructure's Overview) is not one of its contents. + if (!sub.icon || sub.href === item.href || seen.has(sub.icon)) continue seen.add(sub.icon) unique.push(sub) } @@ -311,7 +312,7 @@ function NavRow({ // preview is all or nothing. Six fits both sections we ship (Explore // with Agent Sessions on, Infrastructure) at the tightened gap below. return unique.length > NAV_PREVIEW_MAX_GLYPHS ? undefined : unique - }, [isOpen, item.subItems]) + }, [isOpen, item.subItems, item.href]) // The sub-list can't render at 48px, so the rail turns the row into a menu. // Without this, every child route is stranded while the sidebar is collapsed diff --git a/apps/web/src/components/dashboard/nav-items.test.ts b/apps/web/src/components/dashboard/nav-items.test.ts index 9597edf973..68576b8227 100644 --- a/apps/web/src/components/dashboard/nav-items.test.ts +++ b/apps/web/src/components/dashboard/nav-items.test.ts @@ -115,11 +115,12 @@ describe("navGroups", () => { } }) - it("keeps Infrastructure at six children with six unique glyphs", () => { + it("keeps Infrastructure at six sources with six unique glyphs", () => { // Six is exactly NavRow's all-or-nothing preview cap, so a seventh unique // glyph here would drop the closed row's miniatures entirely rather than // truncate them. A new Kubernetes view goes in `views` (free); a new - // child with a new glyph is not free. + // child with a new glyph is not free. Overview is the section's own page + // and stays out of the preview. const infra = findItem("Infrastructure") expect(infra.subItems?.map((sub) => sub.title)).toEqual([ "Overview", @@ -128,8 +129,10 @@ describe("navGroups", () => { "Cloudflare", "PlanetScale", "Railway", + "Google Cloud", ]) - expect(new Set(infra.subItems?.map((sub) => sub.icon)).size).toBe(NAV_PREVIEW_MAX_GLYPHS) + const sources = infra.subItems?.filter((sub) => sub.href !== infra.href) + expect(new Set(sources?.map((sub) => sub.icon)).size).toBe(NAV_PREVIEW_MAX_GLYPHS) }) it("folds the Kubernetes views behind one row that points at the section root", () => { @@ -199,7 +202,7 @@ describe("partitionInfraSubItems", () => { it("shows every child while the org's surfaces are unknown", () => { const { shown, hidden } = partitionInfraSubItems(subItems(), null, "/infra") - expect(shown).toHaveLength(6) + expect(shown).toHaveLength(7) expect(hidden).toEqual([]) }) @@ -210,7 +213,7 @@ describe("partitionInfraSubItems", () => { "/infra", ) expect(titles(shown)).toEqual(["Overview", "Hosts", "PlanetScale"]) - expect(titles(hidden)).toEqual(["Kubernetes", "Cloudflare", "Railway"]) + expect(titles(hidden)).toEqual(["Kubernetes", "Cloudflare", "Railway", "Google Cloud"]) }) // Containers is a view of Hosts, so a Docker-only org still gets the row. diff --git a/apps/web/src/components/dashboard/nav-items.ts b/apps/web/src/components/dashboard/nav-items.ts index 360e840407..edeeaa5f90 100644 --- a/apps/web/src/components/dashboard/nav-items.ts +++ b/apps/web/src/components/dashboard/nav-items.ts @@ -7,6 +7,7 @@ import { CodeIcon, ComputerIcon, FileIcon, + GoogleCloudIcon, GridIcon, GridSquareCirclePlusIcon, HouseIcon, @@ -27,8 +28,9 @@ import type { OrganizationFeatureFlags } from "@/lib/organization-feature-flags" /** * What a nav child needs from the org before it's worth a row. The five OTel - * surfaces come from the warehouse presence probe; Cloudflare, PlanetScale and - * Railway are integration pages, so their gate is whether the integration is connected. + * surfaces come from the warehouse presence probe; Cloudflare, PlanetScale, + * Railway and Google Cloud are integration pages, so their gate is whether the + * integration is connected (for Google Cloud: with metrics switched on). */ export type NavSurface = | "hosts" @@ -39,6 +41,7 @@ export type NavSurface = | "cloudflare" | "planetscale" | "railway" + | "gcp" export interface NavSubItem { title: string @@ -101,7 +104,8 @@ const overviewItem: NavItem = { * connected; the ones it doesn't have are offered on the Overview instead of * padded into the nav. Containers folds into Hosts as a view, the way Pods, * Nodes and Workloads fold into Kubernetes, and stays typeable in the palette - * through `views`. Six children with six unique marks fits `NavRow`'s preview cap. + * through `views`. The six sources' marks are exactly `NavRow`'s preview cap, which + * leaves the Overview row's own glyph out. */ const infrastructureItem: NavItem = { title: "Infrastructure", @@ -134,6 +138,7 @@ const infrastructureItem: NavItem = { surfaces: ["planetscale"], }, { title: "Railway", href: "/infra/railway", icon: RailwayIcon, surfaces: ["railway"] }, + { title: "Google Cloud", href: "/infra/gcp", icon: GoogleCloudIcon, surfaces: ["gcp"] }, ], } diff --git a/apps/web/src/components/icons/cloud-run.tsx b/apps/web/src/components/icons/cloud-run.tsx new file mode 100644 index 0000000000..a6a926b3d1 --- /dev/null +++ b/apps/web/src/components/icons/cloud-run.tsx @@ -0,0 +1,27 @@ +import type { IconProps } from "./icon" + +/** + * Cloud Run's mark in one colour: a chevron and a triangle, their upper halves at reduced + * opacity where the brand mark is lighter. + */ +function CloudRunIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} + +export { CloudRunIcon } diff --git a/apps/web/src/components/icons/google-pubsub.tsx b/apps/web/src/components/icons/google-pubsub.tsx new file mode 100644 index 0000000000..deb5758dc7 --- /dev/null +++ b/apps/web/src/components/icons/google-pubsub.tsx @@ -0,0 +1,21 @@ +import type { IconProps } from "./icon" + +// Source: simple-icons (CC0) — https://simpleicons.org/icons/googlepubsub +function GooglePubSubIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} + +export { GooglePubSubIcon } diff --git a/apps/web/src/components/icons/mysql-dolphin.tsx b/apps/web/src/components/icons/mysql-dolphin.tsx new file mode 100644 index 0000000000..dce4394466 --- /dev/null +++ b/apps/web/src/components/icons/mysql-dolphin.tsx @@ -0,0 +1,22 @@ +import type { IconProps } from "./icon" + +// Source: devicon (MIT) — https://github.com/devicons/devicon, `mysql-original`, coordinates rounded. +// The dolphin alone: the wordmark of `MysqlIcon` is unreadable at row size. +function MysqlDolphinIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} + +export { MysqlDolphinIcon } diff --git a/apps/web/src/components/icons/parentheses-dots.tsx b/apps/web/src/components/icons/parentheses-dots.tsx new file mode 100644 index 0000000000..9c1f4e24fa --- /dev/null +++ b/apps/web/src/components/icons/parentheses-dots.tsx @@ -0,0 +1,31 @@ +import type { IconProps } from "./icon" + +const paths: ReadonlyArray = [ + "M7 4Q3 12 7 20", + "M17 4Q21 12 17 20", + "M9 12H9.01", + "M12 12H12.01", + "M15 12H15.01", +] + +/** A function call, `(···)`. */ +function ParenthesesDotsIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} + +export { ParenthesesDotsIcon } diff --git a/apps/web/src/components/icons/sql-server.tsx b/apps/web/src/components/icons/sql-server.tsx new file mode 100644 index 0000000000..f91b5e6631 --- /dev/null +++ b/apps/web/src/components/icons/sql-server.tsx @@ -0,0 +1,21 @@ +import type { IconProps } from "./icon" + +// Source: devicon (MIT) — https://github.com/devicons/devicon, `microsoftsqlserver-plain`, coordinates rounded +function SqlServerIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} + +export { SqlServerIcon } diff --git a/apps/web/src/components/infra/chart-utils.test.ts b/apps/web/src/components/infra/chart-utils.test.ts index 2d1fa64640..f1e6e48b3a 100644 --- a/apps/web/src/components/infra/chart-utils.test.ts +++ b/apps/web/src/components/infra/chart-utils.test.ts @@ -10,6 +10,8 @@ describe("formatValueWithUnit", () => { it("renders cores with a unit suffix (never a bare number)", () => { expect(formatValueWithUnit(0.067, "cores")).toBe("0.067 cores") expect(formatValueWithUnit(2, "cores")).toBe("2 cores") + expect(formatValueWithUnit(0.00042, "cores")).toBe("0.00042 cores") + expect(formatValueWithUnit(0, "cores")).toBe("0 cores") }) it("renders a duration for seconds", () => { @@ -25,6 +27,11 @@ describe("formatValueWithUnit", () => { expect(formatValueWithUnit(2048, "bytes_per_second")).toBe("2.0 KB/s") }) + it("renders a count at reading precision, with no unit", () => { + expect(formatValueWithUnit(1.5, "count")).toBe("1.5") + expect(formatValueWithUnit(12_340, "count")).toBe("12.3K") + }) + it("guards against non-finite values", () => { expect(formatValueWithUnit(Number.NaN, "cores")).toBe("—") expect(formatValueWithUnit(Number.POSITIVE_INFINITY, "percent")).toBe("—") diff --git a/apps/web/src/components/infra/chart-utils.ts b/apps/web/src/components/infra/chart-utils.ts index c399473b0c..e2f222a6b7 100644 --- a/apps/web/src/components/infra/chart-utils.ts +++ b/apps/web/src/components/infra/chart-utils.ts @@ -11,8 +11,8 @@ import { formatBytesPerSecond, formatLatency, formatLoad, - formatNumber, formatPercent, + formatRate, formatThroughput, } from "@maple/ui/lib/format" import { formatTimeInTimezone } from "@/lib/timezone-format" @@ -41,7 +41,8 @@ export type ChartUnit = // whole seconds and renders anything at or below zero as an em dash. | "milliseconds" | "rate" - // A per-bucket event count (requests, invocations), compact like "1.2K". + // A quantity: per-bucket event counts (requests, invocations) or levels (instances, + // connections), compact like "1.2K" and fractional below one. | "count" /** Compact, human duration ("45s", "12m", "3h 20m", "2d 4h"). */ @@ -67,7 +68,8 @@ export function formatValueWithUnit(value: number, unit: ChartUnit): string { case "percent": return formatPercent(value) case "cores": - return `${value.toLocaleString(undefined, { maximumFractionDigits: 3 })} cores` + // A sidecar idles at a fraction of a millicore, which three decimals round to zero. + return `${value.toLocaleString(undefined, Math.abs(value) < 0.01 ? { maximumSignificantDigits: 2 } : { maximumFractionDigits: 3 })} cores` case "seconds": return formatSeconds(value) case "load": @@ -81,7 +83,7 @@ export function formatValueWithUnit(value: number, unit: ChartUnit): string { case "rate": return formatThroughput(value, "/s") case "count": - return formatNumber(value) + return formatRate(value) } } diff --git a/apps/web/src/components/infra/gcp/charts.test.ts b/apps/web/src/components/infra/gcp/charts.test.ts new file mode 100644 index 0000000000..5d8c9448a6 --- /dev/null +++ b/apps/web/src/components/infra/gcp/charts.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it } from "vitest" + +import { GCP_INFRA_SERVICE_IDS, gcpInfraMetrics } from "@maple/domain/gcp-infra" +import { + GCP_GKE_NODE_CHARTS, + GCP_INFRA_CHARTS, + gcpBuckets, + gcpChartMetric, + gcpClassRows, + gcpLineRows, + gcpWindowPoints, + type GcpBucketPoint, + type GcpChart, +} from "./charts" +import { gcpWorkload, type GcpMetricReader } from "./tabs" + +const T1 = "2026-10-08T12:00:00.000Z" +const T2 = "2026-10-08T12:05:00.000Z" +/** A window that holds both buckets whole. */ +const WINDOW = { startTime: "2026-10-08 12:00:00", endTime: "2026-10-08 12:10:00", bucketSeconds: 300 } + +const point = ( + bucket: string, + metric: string, + label: string, + total: number, + samples = 1, +): GcpBucketPoint => ({ + bucket, + metric, + label, + total, + samples, +}) + +/** The metrics a chart reads, by running its series against a reader that records them. */ +const metricsOf = (chart: GcpChart): ReadonlyArray => { + if (chart.kind === "classes") return [chart.metric] + const read: Array = [] + const recorder: GcpMetricReader = { + total: (metric) => { + read.push(metric) + return 0 + }, + mean: (metric) => { + read.push(metric) + return undefined + }, + } + for (const series of chart.series) { + series.value(recorder, 300) + read.push(series.metric) + } + return read +} + +const lineChart = (service: (typeof GCP_INFRA_SERVICE_IDS)[number], title: string) => { + const chart = GCP_INFRA_CHARTS[service].find((candidate) => candidate.title === title) + if (chart?.kind !== "lines") throw new Error(`no line chart ${title}`) + return chart +} + +describe("GCP_INFRA_CHARTS", () => { + it("reads only metrics the page's query returns", () => { + for (const service of GCP_INFRA_SERVICE_IDS) { + const known = new Set(gcpInfraMetrics(service).map((metric) => metric.name)) + const read = GCP_INFRA_CHARTS[service].flatMap(metricsOf) + expect(read.length, service).toBeGreaterThan(0) + expect( + read.filter((metric) => !known.has(metric)), + service, + ).toEqual([]) + } + const nodes = new Set(gcpInfraMetrics("gkeNodes").map((metric) => metric.name)) + expect(GCP_GKE_NODE_CHARTS.flatMap(metricsOf).filter((metric) => !nodes.has(metric))).toEqual([]) + }) + + it("titles every chart uniquely on its page", () => { + for (const service of GCP_INFRA_SERVICE_IDS) { + const titles = GCP_INFRA_CHARTS[service].map((chart) => chart.title) + expect(new Set(titles).size, service).toBe(titles.length) + } + }) + + it("opens a counter as a sum and a gauge or percentile as a gauge", () => { + // The explorer keeps apart what the chart draws apart: the classes, the percentiles. + expect(gcpChartMetric(GCP_INFRA_CHARTS.cloudRun[0], [])).toEqual({ + name: "gcp.cloud_run.requests", + type: "sum", + groupBy: "attr.gcp.metric.labels.response_code_class", + }) + expect(gcpChartMetric(lineChart("cloudRun", "Request latency"), [])).toEqual({ + name: "gcp.cloud_run.request.duration", + type: "gauge", + groupBy: "attr.quantile", + }) + expect(gcpChartMetric(lineChart("computeEngine", "Network bytes per second"), [])).toEqual({ + name: "gcp.gce.network.io", + type: "sum", + groupBy: "attr.network.io.direction", + }) + expect(gcpChartMetric(lineChart("cloudSql", "CPU utilization"), []).groupBy).toBeUndefined() + }) +}) + +describe("gcpLineRows", () => { + it("draws a percentile per series in milliseconds and leaves a gap where it did not report", () => { + const rows = gcpLineRows( + lineChart("cloudRun", "Request latency").series, + [ + point(T1, "gcp.cloud_run.request.duration", "0.5", 0.5, 4), + point(T1, "gcp.cloud_run.request.duration", "0.99", 1.5, 2), + point(T2, "gcp.cloud_run.request.duration", "0.5", 0.25, 2), + ], + WINDOW, + ) + expect(rows).toEqual([ + { bucket: T1, attributeValue: "p50", value: 125 }, + { bucket: T1, attributeValue: "p99", value: 750 }, + { bucket: T2, attributeValue: "p50", value: 125 }, + ]) + }) + + it("reads a counter per second, zero in a bucket where the workload reported but counted nothing", () => { + const rows = gcpLineRows( + lineChart("computeEngine", "Network bytes per second").series, + [ + point(T1, "gcp.gce.network.io", "receive", 600_000, 5), + point(T2, "gcp.gce.cpu.utilization", "", 0.5, 5), + ], + WINDOW, + ) + expect(rows).toEqual([ + { bucket: T1, attributeValue: "Received", value: 2000 }, + { bucket: T1, attributeValue: "Sent", value: 0 }, + { bucket: T2, attributeValue: "Received", value: 0 }, + { bucket: T2, attributeValue: "Sent", value: 0 }, + ]) + }) + + it("divides a counter by the part of an edge bucket the window covers", () => { + const received = [point(T1, "gcp.gce.network.io", "receive", 120_000, 2)] + const rows = gcpLineRows(lineChart("computeEngine", "Network bytes per second").series, received, { + ...WINDOW, + // The window opens three minutes into the bucket. + startTime: "2026-10-08 12:03:00", + }) + expect(rows[0]).toEqual({ bucket: T1, attributeValue: "Received", value: 1000 }) + // A window that ends seconds into its last bucket still divides by one point's minute. + expect( + gcpLineRows(lineChart("computeEngine", "Network bytes per second").series, received, { + ...WINDOW, + endTime: "2026-10-08 12:00:10", + })[0]?.value, + ).toBe(2000) + }) + + it("turns CPU seconds per minute into cores and a lag in seconds into milliseconds", () => { + expect( + gcpLineRows( + lineChart("gke", "CPU cores").series, + [point(T1, "gcp.gke.container.cpu.time", "", 150, 5)], + WINDOW, + ), + ).toEqual([{ bucket: T1, attributeValue: "Usage", value: 0.5 }]) + expect( + gcpLineRows( + lineChart("cloudSql", "Replica lag").series, + [point(T1, "gcp.cloud_sql.replication.lag", "", 2.5, 5)], + WINDOW, + ), + ).toEqual([{ bucket: T1, attributeValue: "Lag", value: 500 }]) + }) + + it("draws an error rate only where there was traffic", () => { + const rows = gcpLineRows( + lineChart("loadBalancing", "5xx rate").series, + [ + point(T1, "gcp.load_balancing.requests", "200", 90), + point(T1, "gcp.load_balancing.requests", "500", 10), + point(T2, "gcp.load_balancing.request.duration", "0.5", 10), + ], + WINDOW, + ) + expect(rows).toEqual([{ bucket: T1, attributeValue: "5xx rate", value: 0.1 }]) + }) +}) + +describe("gcpClassRows", () => { + it("draws a counter per second by class, zero where a class counted nothing", () => { + const rows = gcpClassRows( + "gcp.load_balancing.requests", + [ + point(T1, "gcp.load_balancing.requests", "200", 900), + point(T1, "gcp.load_balancing.requests", "500", 30), + point(T1, "gcp.load_balancing.requests", "0", 3), + point(T2, "gcp.load_balancing.request.duration", "0.5", 10), + ], + WINDOW, + ) + expect(rows).toEqual([ + { bucket: T1, attributeValue: "2xx", value: 3 }, + { bucket: T1, attributeValue: "5xx", value: 0.1 }, + { bucket: T1, attributeValue: "unknown", value: 0.01 }, + { bucket: T2, attributeValue: "2xx", value: 0 }, + { bucket: T2, attributeValue: "5xx", value: 0 }, + { bucket: T2, attributeValue: "unknown", value: 0 }, + ]) + }) + + it("is empty for a counter that never counted", () => { + expect( + gcpClassRows( + "gcp.pubsub.subscription.push_requests", + [point(T1, "gcp.pubsub.subscription.undelivered_messages", "", 4)], + WINDOW, + ), + ).toEqual([]) + }) +}) + +describe("gcpBuckets", () => { + const T4 = "2026-10-08T12:15:00.000Z" + const idle = [ + point(T1, "gcp.cloud_run.requests", "2xx", 300), + point(T1, "gcp.cloud_run.request.duration", "0.5", 0.5, 4), + // The service scaled to zero and reported nothing for two buckets. + point(T4, "gcp.cloud_run.requests", "2xx", 600), + point(T4, "gcp.cloud_run.request.duration", "0.5", 1, 4), + ] + const window = { ...WINDOW, endTime: "2026-10-08 12:20:00" } + + it("keeps the silent buckets between the first and the last that reported", () => { + expect(gcpBuckets(idle, 300).map(({ bucket, points }) => [bucket, points.length])).toEqual([ + [T1, 2], + [T2, 0], + ["2026-10-08T12:10:00.000Z", 0], + [T4, 2], + ]) + expect(gcpBuckets([], 300)).toEqual([]) + }) + + it("reads a counter as zero through the silence and leaves a gauge out of it", () => { + const requests = gcpClassRows("gcp.cloud_run.requests", idle, window) + expect(requests.map((row) => row.value)).toEqual([1, 0, 0, 2]) + const latency = gcpLineRows(lineChart("cloudRun", "Request latency").series, idle, window) + expect(latency.map((row) => [row.bucket, row.value])).toEqual([ + [T1, 125], + [T4, 250], + ]) + }) +}) + +describe("gcpWindowPoints", () => { + const points = [ + point(T2, "gcp.cloud_run.requests", "2xx", 40), + point(T1, "gcp.cloud_run.requests", "2xx", 50), + point(T1, "gcp.cloud_run.requests", "5xx", 10), + point(T1, "gcp.cloud_run.request.duration", "0.95", 0.5, 2), + point(T2, "gcp.cloud_run.request.duration", "0.95", 0.25, 2), + ] + + it("orders buckets oldest first", () => { + expect(gcpBuckets(points, 300).map(({ bucket }) => bucket)).toEqual([T1, T2]) + }) + + it("folds the buckets into the numbers of the workload's table row", () => { + const workload = gcpWorkload("cloudRun", ["api"], gcpWindowPoints(points)) + // Requests, 5xx rate, latency p95: a total, a share of the total, a mean over every point. + expect(workload.values.slice(0, 3)).toEqual([100, 0.1, 0.1875]) + expect(workload.errors).toBe("crit") + }) +}) diff --git a/apps/web/src/components/infra/gcp/charts.ts b/apps/web/src/components/infra/gcp/charts.ts new file mode 100644 index 0000000000..9ee758b3c1 --- /dev/null +++ b/apps/web/src/components/infra/gcp/charts.ts @@ -0,0 +1,394 @@ +// A Google Cloud workload's page: which charts each service shows, and how the buckets of the +// timeseries query become their rows. Pure (no React, no atoms). + +import type { GcpInfraServiceId } from "@maple/domain/gcp-infra" +import { GCP_METRIC_GROUPS, gcpMetricPointAttributes } from "@maple/domain/gcp-metrics" +import { toEpochMs } from "@maple/ui/lib/time-format" + +import type { ChartUnit } from "../chart-utils" +import { + failedShare, + gcpMetricReader, + serverErrors, + unhealthy, + type GcpMetricReader, + type GcpMetricValue, +} from "./tabs" + +/** A row of the timeseries query: see `gcpInfraTimeseriesSQL`. */ +export interface GcpBucketPoint extends GcpMetricValue { + readonly bucket: string +} + +export interface GcpSeries { + readonly name: string + /** The metric it reads, which is where the chart's link into the metrics explorer opens. */ + readonly metric: string + readonly value: (read: GcpMetricReader, seconds: number) => number | undefined +} + +interface GcpChartBase { + readonly title: string + /** Why a workload may never report it, shown in place of an empty plot. */ + readonly note?: string +} + +export type GcpChart = + | (GcpChartBase & { + readonly kind: "lines" + readonly unit: ChartUnit + readonly series: ReadonlyArray + /** Draws the 80% rule: for a share of a limit or of capacity. */ + readonly threshold?: boolean + }) + /** A counter per second, one line per value of its label. */ + | (GcpChartBase & { readonly kind: "classes"; readonly metric: string }) + +const scaled = (value: number | undefined, scale: number) => (value === undefined ? undefined : value * scale) + +/** A gauge's or percentile's mean over the bucket. */ +const mean = (name: string, metric: string, label?: string, scale = 1): GcpSeries => ({ + name, + metric, + value: (read) => scaled(read.mean(metric, label), scale), +}) + +/** + * A counter per second, of one label value when given. The poller stores no zero deltas, so a + * quiet bucket reads as zero. + */ +const rate = (name: string, metric: string, label?: string): GcpSeries => ({ + name, + metric, + value: (read, seconds) => read.total(metric, label) / seconds, +}) + +/** A counter's sum over the bucket. */ +const count = (name: string, metric: string): GcpSeries => ({ + name, + metric, + value: (read) => read.total(metric), +}) + +/** The series builders, for a page that reads one metric of a chart on its own. */ +export const gcpSeries = { mean, rate, count } as const + +const percentiles = (metric: string, scale = 1): ReadonlyArray => [ + mean("p50", metric, "0.5", scale), + mean("p95", metric, "0.95", scale), + mean("p99", metric, "0.99", scale), +] + +/** A duration stored in seconds, charted in milliseconds: the `seconds` unit rounds to whole ones. */ +const MS = 1000 + +const lines = ( + title: string, + unit: ChartUnit, + series: ReadonlyArray, + options: { readonly threshold?: boolean; readonly note?: string } = {}, +): GcpChart => ({ kind: "lines", title, unit, series, ...options }) + +const classes = (title: string, metric: string, note?: string): GcpChart => ({ + kind: "classes", + title, + metric, + note, +}) + +const OF_LIMIT = { threshold: true } as const + +const RUN = "gcp.cloud_run" +const FUNCTION = "gcp.cloud_functions" +const CONTAINER = "gcp.gke.container" +const NODE = "gcp.gke.node" +const INSTANCE = "gcp.gce" +const DATABASE = "gcp.cloud_sql" +const SUBSCRIPTION = "gcp.pubsub.subscription" +const LB = "gcp.load_balancing" + +const PUSH_ONLY = "Push subscriptions only." + +export const GCP_INFRA_CHARTS: Record> = { + cloudRun: [ + classes("Requests by response class", `${RUN}.requests`), + lines("5xx rate", "percent", [ + { + name: "5xx rate", + metric: `${RUN}.requests`, + value: failedShare(`${RUN}.requests`, serverErrors(`${RUN}.requests`)), + }, + ]), + lines("Request latency", "milliseconds", percentiles(`${RUN}.request.duration`, MS)), + lines("Instances", "count", [ + mean("Active instances", `${RUN}.container.instance.count`, "active"), + mean("Idle instances", `${RUN}.container.instance.count`, "idle"), + ]), + lines("CPU utilization", "percent", percentiles(`${RUN}.container.cpu.utilization`), OF_LIMIT), + lines("Memory utilization", "percent", percentiles(`${RUN}.container.memory.utilization`), OF_LIMIT), + lines( + "Peak concurrency per instance", + "count", + percentiles(`${RUN}.container.request.max_concurrency`), + ), + // Billable seconds per second: how many instances were billed at once. + lines("Billable instances", "count", [rate("Billed", `${RUN}.container.billable_time`)]), + ], + cloudFunctions: [ + classes("Executions by status", `${FUNCTION}.executions`), + lines("Error rate", "percent", [ + { + name: "Error rate", + metric: `${FUNCTION}.executions`, + value: failedShare(`${FUNCTION}.executions`, unhealthy(`${FUNCTION}.executions`, "ok")), + }, + ]), + lines("Execution time", "milliseconds", percentiles(`${FUNCTION}.execution.duration`, MS)), + lines("Instances", "count", [ + mean("Active instances", `${FUNCTION}.instance.count`, "active"), + mean("Idle instances", `${FUNCTION}.instance.count`, "idle"), + ]), + lines("Memory per execution", "bytes", percentiles(`${FUNCTION}.memory.usage`)), + lines("Egress bytes per second", "bytes_per_second", [rate("Sent", `${FUNCTION}.network.io`)]), + ], + gke: [ + // A point is the CPU seconds used in one minute. + lines("CPU cores", "cores", [mean("Usage", `${CONTAINER}.cpu.time`, undefined, 1 / 60)]), + lines("CPU of limit", "percent", [mean("Usage", `${CONTAINER}.cpu.limit.utilization`)], { + ...OF_LIMIT, + note: "Containers with a CPU limit only.", + }), + lines("Memory", "bytes", [ + mean("Non-evictable", `${CONTAINER}.memory.usage`, "non-evictable"), + mean("Evictable cache", `${CONTAINER}.memory.usage`, "evictable"), + ]), + // Non-evictable memory is what counts against the limit. + lines( + "Memory of limit", + "percent", + [mean("Usage", `${CONTAINER}.memory.limit.utilization`, "non-evictable")], + { ...OF_LIMIT, note: "Containers with a memory limit only." }, + ), + lines("Restarts", "count", [count("Restarts", `${CONTAINER}.restarts`)]), + ], + computeEngine: [ + lines("CPU utilization", "percent", [mean("Usage", `${INSTANCE}.cpu.utilization`)], OF_LIMIT), + lines("Memory used", "bytes", [mean("Used", `${INSTANCE}.memory.usage`)], { + note: "E2 machine types only.", + }), + lines("Network bytes per second", "bytes_per_second", [ + rate("Received", `${INSTANCE}.network.io`, "receive"), + rate("Sent", `${INSTANCE}.network.io`, "transmit"), + ]), + lines("Disk bytes per second", "bytes_per_second", [ + rate("Read", `${INSTANCE}.disk.io`, "read"), + rate("Write", `${INSTANCE}.disk.io`, "write"), + ]), + ], + cloudSql: [ + lines("CPU utilization", "percent", [mean("Usage", `${DATABASE}.cpu.utilization`)], OF_LIMIT), + lines("Memory utilization", "percent", [mean("Usage", `${DATABASE}.memory.utilization`)], OF_LIMIT), + lines("Disk utilization", "percent", [mean("Usage", `${DATABASE}.disk.utilization`)], OF_LIMIT), + lines("Connections", "count", [mean("Connections", `${DATABASE}.connection.count`)]), + lines("Disk operations", "rate", [ + rate("Read", `${DATABASE}.disk.operations`, "read"), + rate("Write", `${DATABASE}.disk.operations`, "write"), + ]), + lines("Replica lag", "milliseconds", [mean("Lag", `${DATABASE}.replication.lag`, undefined, MS)], { + note: "Read replicas only.", + }), + ], + pubsub: [ + lines("Backlog", "count", [mean("Unacked messages", `${SUBSCRIPTION}.undelivered_messages`)]), + lines("Unacked message age", "milliseconds", [ + mean("Age", `${SUBSCRIPTION}.oldest_unacked_message.age`, undefined, MS), + ]), + lines("Messages", "rate", [ + rate("Delivered", `${SUBSCRIPTION}.sent_messages`), + rate("Acked", `${SUBSCRIPTION}.acked_messages`), + ]), + lines("Dead-lettered messages", "count", [ + count("Dead-lettered", `${SUBSCRIPTION}.dead_lettered_messages`), + ]), + classes("Push requests by response", `${SUBSCRIPTION}.push_requests`, PUSH_ONLY), + lines( + "Push error rate", + "percent", + [ + { + name: "Push errors", + metric: `${SUBSCRIPTION}.push_requests`, + value: failedShare( + `${SUBSCRIPTION}.push_requests`, + unhealthy(`${SUBSCRIPTION}.push_requests`, "ack"), + ), + }, + ], + { note: PUSH_ONLY }, + ), + ], + loadBalancing: [ + classes("Requests by response class", `${LB}.requests`), + lines("5xx rate", "percent", [ + { + name: "5xx rate", + metric: `${LB}.requests`, + value: failedShare(`${LB}.requests`, serverErrors(`${LB}.requests`)), + }, + ]), + lines("Total latency", "milliseconds", percentiles(`${LB}.request.duration`, MS)), + lines("Backend latency", "milliseconds", percentiles(`${LB}.backend.request.duration`, MS)), + classes("Backend requests by class", `${LB}.backend.requests`), + lines("Traffic bytes per second", "bytes_per_second", [ + rate("Received", `${LB}.network.io`, "receive"), + rate("Sent", `${LB}.network.io`, "transmit"), + ]), + ], +} satisfies Record> + +/** The nodes of a GKE container's cluster, averaged over the nodes: read from the `gkeNodes` source. */ +export const GCP_GKE_NODE_CHARTS: ReadonlyArray = [ + lines("Node CPU", "percent", [mean("Usage", `${NODE}.cpu.allocatable.utilization`)], OF_LIMIT), + lines( + "Node memory", + "percent", + [ + mean("Non-evictable", `${NODE}.memory.allocatable.utilization`, "non-evictable"), + mean("Evictable cache", `${NODE}.memory.allocatable.utilization`, "evictable"), + ], + OF_LIMIT, + ), +] + +/** + * The metric a chart opens in the metrics explorer: the first of its metrics that reported. + * `groupBy` keeps what the chart draws apart there too: the percentiles, or the values of the + * metric's label or direction. + */ +export function gcpChartMetric( + chart: GcpChart, + points: ReadonlyArray, +): { readonly name: string; readonly type: "sum" | "gauge"; readonly groupBy: string | undefined } { + const candidates = chart.kind === "classes" ? [chart.metric] : chart.series.map((series) => series.metric) + const name = candidates.find((metric) => points.some((point) => point.metric === metric)) ?? candidates[0] + const stored = GCP_METRIC_GROUPS.flatMap((group) => group.metrics).find((metric) => metric.name === name) + const label = + stored === undefined + ? undefined + : stored.kind === "quantiles" + ? "quantile" + : gcpMetricPointAttributes(stored)[0] + return { + name, + type: stored?.kind === "sum" ? "sum" : "gauge", + groupBy: label === undefined ? undefined : `attr.${label}`, + } +} + +/** The window a page reads, as its query got it. */ +export interface GcpChartWindow { + readonly startTime: string + readonly endTime: string + readonly bucketSeconds: number +} + +/** + * Every bucket from the first the workload reported in to the last, oldest first, each with its + * points. A workload that scales to zero reports nothing while idle: those buckets are kept, + * empty, so a counter reads zero there and a gauge has a gap instead of a line drawn across. + */ +export function gcpBuckets( + points: ReadonlyArray, + bucketSeconds: number, +): ReadonlyArray<{ readonly bucket: string; readonly points: ReadonlyArray }> { + const byTime = new Map>() + for (const point of points) { + const at = toEpochMs(point.bucket) + const bucket = byTime.get(at) + if (bucket === undefined) byTime.set(at, [point]) + else bucket.push(point) + } + if (byTime.size === 0) return [] + const times = [...byTime.keys()] + const last = Math.max(...times) + const buckets = [] + for (let at = Math.min(...times); at <= last; at += bucketSeconds * 1000) { + buckets.push({ bucket: new Date(at).toISOString(), points: byTime.get(at) ?? [] }) + } + return buckets +} + +/** The whole window, each metric and label value once: what the workload's table row reads. */ +export function gcpWindowPoints(points: ReadonlyArray): ReadonlyArray { + const folded = new Map() + for (const { metric, label, total, samples } of points) { + const id = `${metric}\u0000${label}` + const sum = folded.get(id) + folded.set(id, { + metric, + label, + total: (sum?.total ?? 0) + total, + samples: (sum?.samples ?? 0) + samples, + }) + } + return [...folded.values()] +} + +interface GcpChartRow { + readonly bucket: string + readonly attributeValue: string + readonly value: number +} + +/** + * The seconds of a bucket that lie in the window. The window cuts its first and last bucket + * short, and a rate over the whole bucket would dip there. Never under one point's minute. + */ +const coveredSeconds = (bucket: string, window: GcpChartWindow) => { + const from = toEpochMs(bucket) + const until = Math.min(from + window.bucketSeconds * 1000, toEpochMs(window.endTime)) + return Math.max(60, (until - Math.max(from, toEpochMs(window.startTime))) / 1000) +} + +/** A line chart's rows: one per bucket and series that has a value there. */ +export function gcpLineRows( + series: ReadonlyArray, + points: ReadonlyArray, + window: GcpChartWindow, +): ReadonlyArray { + return gcpBuckets(points, window.bucketSeconds).flatMap(({ bucket, points: bucketPoints }) => { + const read = gcpMetricReader(bucketPoints) + const seconds = coveredSeconds(bucket, window) + return series.flatMap((entry) => { + const value = entry.value(read, seconds) + return value === undefined ? [] : [{ bucket, attributeValue: entry.name, value }] + }) + }) +} + +/** The load balancer writes a response class as "500", everything else as "5xx". */ +const className = (label: string) => + label === "" || label === "0" ? "unknown" : label.replace(/^(\d)00$/, "$1xx") + +/** + * A counter's rows by label value, per second. Every class the window saw has a row in every + * bucket, zero where it counted nothing. + */ +export function gcpClassRows( + metric: string, + points: ReadonlyArray, + window: GcpChartWindow, +): ReadonlyArray { + const counted = points.filter((point) => point.metric === metric) + const names = [...new Set(counted.map((point) => className(point.label)))].sort() + return gcpBuckets(points, window.bucketSeconds).flatMap(({ bucket, points: bucketPoints }) => + names.map((name) => ({ + bucket, + attributeValue: name, + value: + bucketPoints + .filter((point) => point.metric === metric && className(point.label) === name) + .reduce((sum, point) => sum + point.total, 0) / coveredSeconds(bucket, window), + })), + ) +} diff --git a/apps/web/src/components/infra/gcp/cloud-sql-signals.test.ts b/apps/web/src/components/infra/gcp/cloud-sql-signals.test.ts new file mode 100644 index 0000000000..1aefd380cf --- /dev/null +++ b/apps/web/src/components/infra/gcp/cloud-sql-signals.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, it } from "vitest" + +import type { GcpBucketPoint } from "./charts" +import type { GcpCloudSqlInstance } from "./inventory" +import { + gcpAppWait, + gcpCloudSqlHealth, + gcpCloudSqlView, + gcpConnectionPeak, + gcpMatchStatements, + gcpNormalizeStatement, + gcpPoolFull, + gcpQueryInsightsUnavailable, + gcpSqlEngine, + gcpSqlSize, + gcpStatements, + type GcpTracedQuery, +} from "./cloud-sql-signals" + +const BUCKET_SECONDS = 300 +const START_MS = Date.parse("2026-10-14T12:00:00.000Z") +const at = (index: number) => new Date(START_MS + index * BUCKET_SECONDS * 1000).toISOString() +const point = (index: number, metric: string, label: string, total: number, samples = 1): GcpBucketPoint => ({ + bucket: at(index), + metric, + label, + total, + samples, +}) + +const DB = "gcp.cloud_sql" + +const tracedQuery = (statement: string, service: string, p95: number, calls = 1000): GcpTracedQuery => ({ + sampleStatement: statement, + sampleService: service, + serviceCount: 1, + estimatedQueryCount: calls, + p95DurationMs: p95, +}) + +const insight = (queryHash: string, querystring: string, p95LatencyMillis: number | null) => ({ + queryHash, + querystring, + p95LatencyMillis, +}) + +const INSTANCE: GcpCloudSqlInstance = { + id: "acme-prod:orders-db", + name: "orders-db", + databaseVersion: "POSTGRES_15", + edition: "ENTERPRISE", + tier: "db-custom-4-16384", + cpus: 4, + memoryGb: 16, + availabilityType: "REGIONAL", + primary: null, + diskSizeGb: 500, + queryInsightsEnabled: true, + maxConnections: 100, + maxConnectionsSource: "default", +} + +describe("gcpNormalizeStatement", () => { + it("reads Query Insights' placeholders and a span's literals as the same statement", () => { + expect(gcpNormalizeStatement("UPDATE orders SET status = $1, updated_at = $2 WHERE id = $3")).toBe( + gcpNormalizeStatement("update orders set status='paid', updated_at = '2026-10-14' where id=42;"), + ) + }) + + it("reads a driver's ? and an IN list of any length alike", () => { + expect(gcpNormalizeStatement("SELECT * FROM items WHERE id IN (?, ?, ?)")).toBe( + gcpNormalizeStatement("select *\n from items where id in ($1)"), + ) + }) + + it("keeps digits that are part of a name", () => { + expect(gcpNormalizeStatement("SELECT * FROM table1")).toBe("select * from table1") + }) + + it("tells different statements apart", () => { + expect(gcpNormalizeStatement("SELECT * FROM orders WHERE id = $1")).not.toBe( + gcpNormalizeStatement("SELECT * FROM order_items WHERE id = $1"), + ) + }) +}) + +describe("gcpMatchStatements", () => { + const rows = [ + insight("a", "UPDATE orders SET status = $1 WHERE id = $2", 38), + insight("b", "SELECT count(*) FROM orders WHERE created_at > $1", 2800), + ] + + it("joins a statement to the traced shapes of it, the busiest first", () => { + const traced = gcpMatchStatements(rows, [ + tracedQuery("update orders set status = 'paid' where id = 7", "orders-worker", 90, 200), + tracedQuery("UPDATE orders SET status = ? WHERE id = ?", "checkout-api", 1640, 5000), + ]) + expect(traced.get("a")).toEqual({ + services: ["checkout-api", "orders-worker"], + serviceCount: 2, + appP95Ms: 1640, + }) + }) + + it("leaves a statement no shape matches untraced", () => { + const traced = gcpMatchStatements(rows, [tracedQuery("SELECT 1", "checkout-api", 1)]) + expect(traced.has("b")).toBe(false) + }) + + it("ignores shapes without a statement", () => { + expect(gcpMatchStatements([insight("c", "", 1)], [tracedQuery("", "redis-user", 1)]).size).toBe(0) + }) +}) + +describe("gcpAppWait", () => { + const traced = (appP95Ms: number) => + new Map([["a", { services: ["checkout-api"], serviceCount: 1, appP95Ms }]]) + + it("calls out an app side five times the database's and 200ms more", () => { + expect(gcpAppWait([insight("a", "UPDATE orders", 38)], traced(1640))).toEqual({ + service: "checkout-api", + statement: "UPDATE orders", + appP95Ms: 1640, + dbP95Ms: 38, + }) + }) + + it("stays quiet under five times the database's p95", () => { + expect(gcpAppWait([insight("a", "UPDATE orders", 400)], traced(1640))).toBeNull() + }) + + it("stays quiet when the gap is under 200ms", () => { + expect(gcpAppWait([insight("a", "SELECT 1", 2)], traced(150))).toBeNull() + }) + + it("picks the statement the app waits on the most times longer than the database runs it", () => { + const statements = [insight("a", "wider gap", 290), insight("b", "most times", 38)] + const both = new Map([ + ["a", { services: ["checkout-api"], serviceCount: 1, appP95Ms: 1900 }], + ["b", { services: ["checkout-api"], serviceCount: 1, appP95Ms: 1640 }], + ]) + expect(gcpAppWait(statements, both)?.statement).toBe("most times") + }) +}) + +describe("connections", () => { + const points = [ + point(0, `${DB}.connection.count`, "", 60), + point(1, `${DB}.connection.count`, "", 97), + point(1, `${DB}.postgresql.connection.count`, "idle_in_transaction", 38), + point(2, `${DB}.connection.count`, "", 70), + ] + + it("finds the busiest bucket and the idle-in-transaction connections in it", () => { + expect(gcpConnectionPeak(points, BUCKET_SECONDS, 100)).toEqual({ + open: 97, + idleInTransaction: 38, + max: 100, + }) + }) + + it("calls the pool full from 90% of max_connections, never without a max", () => { + expect(gcpPoolFull(gcpConnectionPeak(points, BUCKET_SECONDS, 100))).toBe(true) + expect(gcpPoolFull(gcpConnectionPeak(points, BUCKET_SECONDS, 200))).toBe(false) + expect(gcpPoolFull(gcpConnectionPeak(points, BUCKET_SECONDS, null))).toBe(false) + }) +}) + +describe("gcpCloudSqlHealth", () => { + it("reads PostgreSQL's cache hits, transaction IDs and spills, a missing deadlock count as none", () => { + const points = [ + point(0, `${DB}.postgresql.block_reads`, "buffer_cache", 992), + point(0, `${DB}.postgresql.block_reads`, "disk", 8), + point(0, `${DB}.postgresql.transaction_id.utilization`, "", 0.18), + point(0, `${DB}.postgresql.temp_file.io`, "write", 1.5 * 1024 ** 3), + ] + expect(gcpCloudSqlHealth(points)).toEqual([ + { label: "deadlocks", value: "0" }, + { label: "transaction IDs", value: "18% used", tone: "ok" }, + { label: "cache hit", value: "99.2%", tone: "ok" }, + { label: "temp spill", value: "1.5 GB" }, + ]) + }) + + it("says nothing for a MySQL instance", () => { + expect(gcpCloudSqlHealth([point(0, `${DB}.cpu.utilization`, "", 0.4)])).toEqual([]) + }) +}) + +describe("the instance in words", () => { + it("names the engine and version", () => { + expect(gcpSqlEngine("POSTGRES_15")).toBe("Postgres 15") + expect(gcpSqlEngine("MYSQL_8_0")).toBe("MySQL 8.0") + expect(gcpSqlEngine("SQLSERVER_2022_STANDARD")).toBe("SQL Server 2022 standard") + }) + + it("sizes a tier, a shared core without vCPUs", () => { + expect(gcpSqlSize(INSTANCE)).toBe("4 vCPU, 16 GB") + expect(gcpSqlSize({ ...INSTANCE, cpus: null, memoryGb: 0.6 })).toBe("shared core") + expect(gcpSqlSize({ ...INSTANCE, tier: "db-unknown", cpus: null, memoryGb: null })).toBe("db-unknown") + }) +}) + +describe("gcpCloudSqlView", () => { + it("puts the inventory, databases and replicas in the rail before the disk", () => { + const view = gcpCloudSqlView({ + instance: INSTANCE, + statements: [], + tracedQueries: [], + databases: ["orders", "orders_audit"], + replicas: [{ name: "orders-db-replica-1", lagSeconds: 4 }], + points: [ + point(0, `${DB}.disk.usage`, "", 355 * 1024 ** 3), + point(0, `${DB}.disk.limit`, "", 500 * 1024 ** 3), + ], + bucketSeconds: BUCKET_SECONDS, + }) + expect(view.facts).toEqual([ + { label: "engine", value: "Postgres 15 · Enterprise" }, + { label: "databases", value: "orders · orders_audit" }, + { label: "read replica", value: "orders-db-replica-1" }, + { label: "replica lag", value: "4.00s" }, + { label: "disk", value: "355 of 500 GB", tone: "warn" }, + ]) + }) + + it("focuses the service map on the caller of the busiest traced statement", () => { + const view = gcpCloudSqlView({ + instance: INSTANCE, + statements: [ + { + queryHash: "a", + querystring: "SELECT 1", + calls: 10, + p50LatencyMillis: 1, + p95LatencyMillis: 2, + totalExecutionMillis: 20, + executionShare: 1, + rows: 10, + lockShare: 0, + ioShare: 0, + }, + ], + tracedQueries: [tracedQuery("select 1", "checkout-api", 3)], + databases: undefined, + replicas: [], + points: [], + bucketSeconds: BUCKET_SECONDS, + }) + expect(view.focusService).toBe("checkout-api") + expect(view.wait).toBeNull() + }) +}) + +describe("gcpStatements", () => { + const row = (queryHash: string, executionSeconds: number, calls = 40) => ({ + queryHash, + statement: `SELECT ${queryHash}`, + calls, + executionSeconds, + lockSeconds: executionSeconds / 4, + ioSeconds: executionSeconds / 2, + rowCount: 800, + p50Seconds: 0.002, + p95Seconds: 0.05, + }) + + it("shares each statement's time of the instance's, and its waits of its own", () => { + const [orders] = gcpStatements( + [row("a", 30), row("b", 10)], + [point(0, `${DB}.query.execution.time`, "", 60), point(1, `${DB}.query.execution.time`, "", 40)], + ) + expect(orders).toMatchObject({ + querystring: "SELECT a", + executionShare: 0.3, + lockShare: 0.25, + ioShare: 0.5, + totalExecutionMillis: 30_000, + p95LatencyMillis: 50, + rows: 800, + }) + }) + + it("never shares of less than the statements' own time, and has no latency without calls", () => { + const statements = gcpStatements([row("a", 3), row("b", 1, 0)], []) + expect(statements.map((statement) => statement.executionShare)).toEqual([0.75, 0.25]) + expect(statements[1]).toMatchObject({ p50LatencyMillis: null, p95LatencyMillis: null }) + }) +}) + +describe("gcpQueryInsightsUnavailable", () => { + it("says why from the settings: off, or not PostgreSQL", () => { + expect(gcpQueryInsightsUnavailable(INSTANCE)).toBeNull() + expect(gcpQueryInsightsUnavailable(undefined)).toBeNull() + expect(gcpQueryInsightsUnavailable({ ...INSTANCE, queryInsightsEnabled: false })).toMatch( + /^Query Insights is off for this instance\./, + ) + expect(gcpQueryInsightsUnavailable({ ...INSTANCE, databaseVersion: "MYSQL_8_0" })).toBe( + "Maple reads Query Insights for PostgreSQL instances only. orders-db runs MySQL 8.0.", + ) + }) +}) diff --git a/apps/web/src/components/infra/gcp/cloud-sql-signals.ts b/apps/web/src/components/infra/gcp/cloud-sql-signals.ts new file mode 100644 index 0000000000..c6262fdf6f --- /dev/null +++ b/apps/web/src/components/infra/gcp/cloud-sql-signals.ts @@ -0,0 +1,363 @@ +// A Cloud SQL instance in words and figures: its settings from the inventory, Query Insights +// statements matched to the traces' query shapes, when the app-vs-database callout shows, and the +// rail's health readings. Pure (no React, no atoms). + +import type { GcpCloudSqlTopQueriesResponse } from "@maple/domain/http" +import { formatBytes, formatLatency, formatPercent } from "@maple/ui/lib/format" + +import { gcpBuckets, gcpWindowPoints, type GcpBucketPoint } from "./charts" +import type { GcpCloudSqlInstance } from "./inventory" +import { + GCP_IDLE_IN_TRANSACTION, + GCP_LIMIT_FRACTION, + GCP_WRAPAROUND_FRACTION, + gcpSqlEngineOf, + type GcpReplica, +} from "./list-model" +import { gcpMetricReader } from "./tabs" +import { gcpRailFacts, type GcpRailFact } from "./workload-signals" + +const DATABASE = "gcp.cloud_sql" + +const ENGINES: ReadonlyArray = [ + ["POSTGRES", "Postgres"], + ["MYSQL", "MySQL"], + ["SQLSERVER", "SQL Server"], +] + +/** `POSTGRES_15` as "Postgres 15", `MYSQL_8_0` as "MySQL 8.0", `SQLSERVER_2022_STANDARD` as "SQL Server 2022 standard". */ +export function gcpSqlEngine(databaseVersion: string | null): string | undefined { + if (databaseVersion === null) return undefined + const [prefix, ...version] = databaseVersion.split("_") + const name = ENGINES.find(([engine]) => engine === prefix)?.[1] + if (name === undefined) return databaseVersion + if (prefix !== "SQLSERVER") return [name, version.join(".")].filter(Boolean).join(" ") + const [year, ...edition] = version + return [name, year, edition.join(" ").toLowerCase()].filter(Boolean).join(" ") +} + +const gigabytes = (value: number) => value.toLocaleString(undefined, { maximumFractionDigits: 2 }) + +/** "4 vCPU, 16 GB", "shared core" for a shared-core tier, else the tier's name. */ +export function gcpSqlSize(instance: GcpCloudSqlInstance): string | undefined { + if (instance.memoryGb === null) return instance.tier ?? undefined + return instance.cpus === null + ? "shared core" + : `${instance.cpus} vCPU, ${gigabytes(instance.memoryGb)} GB` +} + +/** "Enterprise", "Enterprise Plus". */ +export const gcpSqlEdition = (edition: string | null): string | undefined => + edition === null + ? undefined + : edition + .split("_") + .map((word) => word.charAt(0) + word.slice(1).toLowerCase()) + .join(" ") + +/** The console page where Query Insights is turned on for an instance. */ +export const gcpQueryInsightsConsoleUrl = (project: string, instance: string): string => + `https://console.cloud.google.com/sql/instances/${encodeURIComponent(instance)}/insights?project=${encodeURIComponent(project)}` + +// Statements ------------------------------------------------------------------------------------- + +/** A Query Insights statement over the window, as the top queries table shows it. */ +export interface GcpStatement { + readonly queryHash: string + /** Normalized by Query Insights: literals are `$1`, `$2`, ... */ + readonly querystring: string + readonly calls: number + /** Null without calls in the window. */ + readonly p50LatencyMillis: number | null + readonly p95LatencyMillis: number | null + readonly totalExecutionMillis: number + /** Of the instance's execution time in the window, 0-1. */ + readonly executionShare: number + readonly rows: number + /** Of the statement's execution time, 0-1. */ + readonly lockShare: number + readonly ioShare: number +} + +/** The top queries panel: the statements, or why Query Insights has none for the instance. */ +export interface GcpTopQueries { + readonly rows: ReadonlyArray + readonly unavailableReason: string | null +} + +/** + * The stored statements as the table reads them, each one's time a share of the instance's in + * the window (`points`, its metrics): every statement's, and never less than the rows' own. + */ +export function gcpStatements( + rows: GcpCloudSqlTopQueriesResponse["data"], + points: ReadonlyArray, +): ReadonlyArray { + const whole = Math.max( + gcpMetricReader(gcpWindowPoints(points)).total(`${DATABASE}.query.execution.time`), + rows.reduce((sum, row) => sum + row.executionSeconds, 0), + ) + const share = (part: number, of: number) => (of > 0 ? part / of : 0) + return rows.map((row) => ({ + queryHash: row.queryHash, + querystring: row.statement, + calls: row.calls, + p50LatencyMillis: row.calls > 0 ? row.p50Seconds * 1000 : null, + p95LatencyMillis: row.calls > 0 ? row.p95Seconds * 1000 : null, + totalExecutionMillis: row.executionSeconds * 1000, + executionShare: share(row.executionSeconds, whole), + rows: row.rowCount, + lockShare: share(row.lockSeconds, row.executionSeconds), + ioShare: share(row.ioSeconds, row.executionSeconds), + })) +} + +/** Why Query Insights has no statements for an instance, when its settings say so. */ +export function gcpQueryInsightsUnavailable(instance: GcpCloudSqlInstance | undefined): string | null { + if (instance === undefined) return null + if (instance.databaseVersion !== null && !instance.databaseVersion.startsWith("POSTGRES")) { + return `Maple reads Query Insights for PostgreSQL instances only. ${instance.name} runs ${gcpSqlEngine(instance.databaseVersion)}.` + } + return instance.queryInsightsEnabled + ? null + : `Query Insights is off for this instance. Turn it on in ${instance.name}'s Query Insights settings in the Google Cloud console; statements appear from then on.` +} + +/** + * A statement as both ends can agree on it: Query Insights writes literals as `$1`, a span may + * carry them or a driver's `?`. Lower case, every literal and placeholder `?`, an `IN` list of any + * length one `?`, whitespace collapsed and dropped around punctuation, no trailing semicolon. + */ +export const gcpNormalizeStatement = (statement: string): string => + statement + .toLowerCase() + .replace(/'(?:[^']|'')*'/g, "?") + .replace(/\$\d+/g, "?") + .replace(/\b\d+(?:\.\d+)?\b/g, "?") + .replace(/\s+/g, " ") + .replace(/\s*([(),=])\s*/g, "$1") + .replace(/\?(?:,\?)+/g, "?") + .replace(/[\s;]+$/, "") + .trim() + +/** A query pattern the traces recorded against the instance's databases: see `serviceDbTopQueriesSQL`. */ +export interface GcpTracedQuery { + readonly sampleStatement: string + readonly sampleService: string + readonly serviceCount: number + readonly estimatedQueryCount: number + readonly p95DurationMs: number +} + +/** A Query Insights statement as the traces saw it. */ +export interface GcpTracedStatement { + /** The services seen calling it, the busiest first. */ + readonly services: ReadonlyArray + /** How many services called it: at least `services.length`. */ + readonly serviceCount: number + /** The client spans' p95, where it is called most. */ + readonly appP95Ms: number +} + +/** The traced side of each statement, by `queryHash`; a statement the traces never recorded has none. */ +export function gcpMatchStatements( + rows: ReadonlyArray>, + tracedQueries: ReadonlyArray, +): ReadonlyMap { + const byStatement = new Map>() + for (const query of tracedQueries) { + if (query.sampleStatement === "") continue + const key = gcpNormalizeStatement(query.sampleStatement) + byStatement.set(key, [...(byStatement.get(key) ?? []), query]) + } + const matched = new Map() + for (const row of rows) { + const found = byStatement + .get(gcpNormalizeStatement(row.querystring)) + ?.toSorted((a, b) => b.estimatedQueryCount - a.estimatedQueryCount) + if (found === undefined) continue + const services = [...new Set(found.map((query) => query.sampleService))] + matched.set(row.queryHash, { + services, + serviceCount: Math.max(services.length, ...found.map((query) => query.serviceCount)), + appP95Ms: found[0].p95DurationMs, + }) + } + return matched +} + +/** The app side this many times slower than the database, and by at least `APP_WAIT_MIN_MS`. */ +const APP_WAIT_RATIO = 5 +const APP_WAIT_MIN_MS = 200 + +/** A statement the app waits on far longer than the database runs it. */ +export interface GcpAppWait { + readonly service: string + readonly statement: string + readonly appP95Ms: number + readonly dbP95Ms: number +} + +/** The statement whose app-side p95 is the most times its database-side p95, when one is worth a callout. */ +export function gcpAppWait( + rows: ReadonlyArray>, + traced: ReadonlyMap, +): GcpAppWait | null { + const waits = rows.flatMap((row): ReadonlyArray => { + const app = traced.get(row.queryHash) + const db = row.p95LatencyMillis + if (app === undefined || db === null) return [] + const appP95Ms = app.appP95Ms + return appP95Ms >= APP_WAIT_RATIO * db && appP95Ms - db >= APP_WAIT_MIN_MS + ? [{ service: app.services[0], statement: row.querystring, appP95Ms, dbP95Ms: db }] + : [] + }) + return waits.toSorted((a, b) => b.appP95Ms / b.dbP95Ms - a.appP95Ms / a.dbP95Ms)[0] ?? null +} + +// Connections ------------------------------------------------------------------------------------ + +/** The window's busiest moment for connections. */ +export interface GcpConnectionPeak { + readonly open: number + /** Of those, PostgreSQL connections holding a transaction open and doing nothing. */ + readonly idleInTransaction: number | undefined + /** `max_connections`; null when the inventory does not know it. */ + readonly max: number | null +} + +/** The bucket with the most open connections, and how many of them sat idle in a transaction. */ +export function gcpConnectionPeak( + points: ReadonlyArray, + bucketSeconds: number, + max: number | null, +): GcpConnectionPeak | undefined { + let peak: GcpConnectionPeak | undefined + for (const { points: bucketPoints } of gcpBuckets(points, bucketSeconds)) { + const read = gcpMetricReader(bucketPoints) + const open = read.mean(`${DATABASE}.connection.count`) + if (open === undefined || (peak !== undefined && open <= peak.open)) continue + peak = { + open, + idleInTransaction: read.mean(`${DATABASE}.postgresql.connection.count`, GCP_IDLE_IN_TRANSACTION), + max, + } + } + return peak +} + +/** The pool ran out: open connections reached 90% of `max_connections`. */ +export const gcpPoolFull = (peak: GcpConnectionPeak | undefined): boolean => + peak !== undefined && peak.max !== null && peak.open >= GCP_LIMIT_FRACTION * peak.max + +// The rail -------------------------------------------------------------------------------------- + +/** + * How healthy PostgreSQL is underneath: deadlocks, transaction IDs used toward wraparound, the + * share of blocks read from the buffer cache, and what spilled to temporary files. A reading the + * instance never reported is left out; MySQL reports none of them. + */ +export function gcpCloudSqlHealth(points: ReadonlyArray): ReadonlyArray { + const read = gcpMetricReader(gcpWindowPoints(points)) + const reported = (metric: string) => points.some((point) => point.metric === metric) + const cached = read.total(`${DATABASE}.postgresql.block_reads`, "buffer_cache") + const blocks = read.total(`${DATABASE}.postgresql.block_reads`) + const transactionIds = read.mean(`${DATABASE}.postgresql.transaction_id.utilization`) + const postgres = transactionIds !== undefined || blocks > 0 + return [ + // A counter stores no zero deltas: on PostgreSQL, none counted is none. + ...(postgres || reported(`${DATABASE}.postgresql.deadlocks`) + ? [{ label: "deadlocks", value: read.total(`${DATABASE}.postgresql.deadlocks`).toLocaleString() }] + : []), + ...(transactionIds === undefined + ? [] + : [ + { + label: "transaction IDs", + value: `${formatPercent(transactionIds)} used`, + // Wraparound is the list's risk from half used on. + tone: + transactionIds >= 0.8 + ? "crit" + : transactionIds >= GCP_WRAPAROUND_FRACTION + ? "warn" + : "ok", + } as const, + ]), + // A tenth of a point matters near 100%: 99.2% and 99.9% are different caches. + ...(blocks > 0 + ? [ + { + label: "cache hit", + value: `${((cached / blocks) * 100).toFixed(1)}%`, + // Higher is better: most reads should come from memory. + tone: cached / blocks >= 0.99 ? "ok" : cached / blocks >= 0.95 ? "warn" : "crit", + } as const, + ] + : []), + ...(postgres + ? [{ label: "temp spill", value: formatBytes(read.total(`${DATABASE}.postgresql.temp_file.io`)) }] + : []), + ] +} + +/** What the inventory and the instance's databases say about it, before its readings. */ +function cloudSqlFacts( + instance: GcpCloudSqlInstance | undefined, + databaseVersion: string | undefined, + databases: ReadonlyArray | undefined, + replicas: ReadonlyArray, +): ReadonlyArray { + const engine = gcpSqlEngine(databaseVersion ?? null) + const edition = instance === undefined ? undefined : gcpSqlEdition(instance.edition) + return [ + ...(engine === undefined + ? [] + : [{ label: "engine", value: edition === undefined ? engine : `${engine} · ${edition}` }]), + ...(databases === undefined || databases.length === 0 + ? [] + : [{ label: "databases", value: databases.join(" · ") }]), + // The name and its lag on lines of their own: a replica's name is long enough to wrap mid-word. + ...replicas.flatMap((replica) => [ + { label: "read replica", value: replica.name }, + ...(replica.lagSeconds === undefined + ? [] + : [{ label: "replica lag", value: formatLatency(replica.lagSeconds * 1000) }]), + ]), + ] +} + +/** + * Everything a Cloud SQL instance's page derives from its reads: each statement's traced side, + * the callout's statement and connections, the rail's facts and health, and the service the + * busiest statement is called from. + */ +export function gcpCloudSqlView(input: { + readonly instance: GcpCloudSqlInstance | undefined + /** Query Insights' statements, by total time; none while they load or can't be read. */ + readonly statements: ReadonlyArray + readonly tracedQueries: ReadonlyArray + readonly databases: ReadonlyArray | undefined + readonly replicas: ReadonlyArray + readonly points: ReadonlyArray + readonly bucketSeconds: number +}) { + const { instance, statements, points, bucketSeconds } = input + const traced = gcpMatchStatements(statements, input.tracedQueries) + return { + traced, + wait: gcpAppWait(statements, traced), + peak: gcpConnectionPeak(points, bucketSeconds, instance?.maxConnections ?? null), + facts: [ + ...cloudSqlFacts( + instance, + instance?.databaseVersion ?? gcpSqlEngineOf(points), + input.databases, + input.replicas, + ), + ...gcpRailFacts("cloudSql", points, bucketSeconds), + ], + health: gcpCloudSqlHealth(points), + focusService: statements.flatMap((row) => traced.get(row.queryHash)?.services ?? [])[0], + } +} diff --git a/apps/web/src/components/infra/gcp/gcp-bands.tsx b/apps/web/src/components/infra/gcp/gcp-bands.tsx new file mode 100644 index 0000000000..97f6862084 --- /dev/null +++ b/apps/web/src/components/infra/gcp/gcp-bands.tsx @@ -0,0 +1,180 @@ +// The band above each Google Cloud list: every row by its worst signal, and one cell per problem +// that scopes the list to it. Counts are over the rows before a scope or search, so the band shows +// what they hide. + +import { TONE_FILL } from "@maple/ui/lib/tone" +import { cn } from "@maple/ui/lib/utils" + +import { FLEET_BAND_BOXED, FleetBand, FleetBandLoading } from "../primitives/fleet-band" +import type { Tone } from "../severity-tokens" +import { + rowHealth, + type GcpDatabaseScope, + type GcpServiceScope, + type GcpSubscriptionScope, + type GcpWorkloadScope, + type RowRules, +} from "./list-model" + +interface CellSpec { + readonly label: string + readonly hint: string + readonly tone: Tone + /** The row's health in the strip's label: "serving 5xx". */ + readonly segment: string +} + +/** What a list's band says: its noun, its honesty line, and a cell per scope. */ +export interface GcpBandSpec { + readonly noun: string + readonly caption: string + readonly cells: Readonly> +} + +export const GCP_SERVICE_BAND: GcpBandSpec = { + noun: "service", + caption: "each service by its worst signal: 5xx, then p95 latency", + cells: { + serving5xx: { + label: "Serving 5xx", + hint: "≥1% of ≥100 requests", + tone: "crit", + segment: "serving 5xx", + }, + slow: { label: "Slow", hint: "latency p95 ≥1s", tone: "warn", segment: "slow" }, + idle: { + label: "Idle, still billed", + hint: "0 requests, instance time > 0", + tone: "neutral", + segment: "idle", + }, + }, +} + +export const GCP_SUBSCRIPTION_BAND: GcpBandSpec = { + noun: "subscription", + caption: "each subscription by its worst signal: backlog, dead letters, age", + cells: { + backlogGrowing: { + label: "Backlog growing", + hint: "unacked up across the window", + tone: "crit", + segment: "backlog growing", + }, + stale: { label: "Stale", hint: "oldest unacked ≥10m", tone: "warn", segment: "stale" }, + deadLettering: { + label: "Dead-lettering", + hint: "messages moved to a DLQ", + tone: "crit", + segment: "dead-lettering", + }, + }, +} + +export const GCP_DATABASE_BAND: GcpBandSpec = { + noun: "instance", + caption: "each instance by its worst signal: a limit, wraparound, then query p95", + cells: { + atLimit: { + label: "At a limit", + hint: "connections, CPU or disk ≥90%", + tone: "crit", + segment: "at a limit", + }, + slow: { label: "Slow", hint: "query p95 ≥1s", tone: "warn", segment: "slow" }, + wraparound: { + label: "Wraparound risk", + hint: "transaction IDs ≥50% used", + tone: "warn", + segment: "wraparound risk", + }, + fillingUp: { + label: "Filling up", + hint: "connections, CPU or disk ≥80%", + tone: "warn", + segment: "filling up", + }, + replicaLag: { + label: "Replica lag", + hint: "a replica ≥10s behind", + tone: "warn", + segment: "replica lag", + }, + }, +} + +export const gcpWorkloadBand = (noun: string): GcpBandSpec => ({ + noun, + caption: `each ${noun} by its busiest share of a limit and its error rate`, + cells: { + atLimit: { + label: "At a limit", + hint: "a share of a limit ≥90%", + tone: "crit", + segment: "at a limit", + }, + hot: { label: "Hot", hint: "a share of a limit ≥60%", tone: "warn", segment: "hot" }, + erroring: { label: "Erroring", hint: "≥5% of ≥100 requests", tone: "crit", segment: "erroring" }, + }, +}) + +/** A row's dot: its worst signal's tone, hollow for a quiet problem, blue when healthy. */ +export function GcpHealthDot({ spec, health }: { spec: GcpBandSpec; health: S | "ok" }) { + const tone = health === "ok" ? "ok" : spec.cells[health].tone + return ( + + ) +} + +export function GcpBand({ + rows, + rules, + spec, + activeScope, + onScopeChange, +}: { + rows: ReadonlyArray + rules: RowRules + spec: GcpBandSpec + activeScope: S | undefined + onScopeChange: (scope: S | undefined) => void +}) { + const healths = rows.map((row) => rowHealth(rules, row)) + const count = (health: S | "ok") => healths.filter((candidate) => candidate === health).length + return ( + + total={rows.length} + noun={spec.noun} + caption={spec.caption} + segments={[ + { key: "healthy", count: count("ok"), className: TONE_FILL.ok }, + // The mildest problem nearest the healthy share, the worst at the end. + ...rules.worst.toReversed().map((scope) => ({ + key: spec.cells[scope].segment, + count: count(scope), + className: TONE_FILL[spec.cells[scope].tone], + })), + ]} + cells={rules.scopes.map((scope) => ({ + scope, + label: spec.cells[scope].label, + hint: spec.cells[scope].hint, + tone: spec.cells[scope].tone, + value: rows.filter((row) => rules.inScope(row, scope)).length, + }))} + activeScope={activeScope} + onScopeChange={onScopeChange} + className={FLEET_BAND_BOXED} + /> + ) +} + +export function GcpBandLoading() { + return +} diff --git a/apps/web/src/components/infra/gcp/gcp-cloudsql-top-queries.tsx b/apps/web/src/components/infra/gcp/gcp-cloudsql-top-queries.tsx new file mode 100644 index 0000000000..3b13aec419 --- /dev/null +++ b/apps/web/src/components/infra/gcp/gcp-cloudsql-top-queries.tsx @@ -0,0 +1,194 @@ +// A Cloud SQL instance's top statements: Query Insights' figures, as the poller stored them, beside +// the traces' view of the same statement. The look of PlanetScale's top queries, as a table. + +import { useState } from "react" + +import { EmptyMessage } from "@maple/ui/components/ui/empty" +import { Meter } from "@maple/ui/components/ui/meter" +import { SkeletonList } from "@maple/ui/components/ui/skeleton" +import { EMPTY_VALUE, formatLatency, formatNumber, formatPercent } from "@maple/ui/lib/format" +import { latencyToneClass } from "@maple/ui/lib/latency-tone" +import { TONE_TEXT } from "@maple/ui/lib/tone" +import { cn } from "@maple/ui/lib/utils" + +import { ColumnHead, DataTable } from "@/components/common/data-table" +import { ErrorState } from "@/components/common/error-state" +import { SectionHeading } from "@/components/common/section-heading" +import { Result } from "@/lib/effect-atom" + +import type { GcpStatement, GcpTopQueries, GcpTracedStatement } from "./cloud-sql-signals" + +const SHOWN = 5 +const STATEMENT = "w-0 min-w-[260px] flex-1" +const NUMBER = "text-right font-mono text-xs tabular-nums" +const SUB = "truncate font-mono text-3xs text-muted-foreground" + +/** Rows read per call from here: a scan, not a lookup. */ +const MANY_ROWS = 10_000 +/** Lock wait from these shares of a statement's time: contention, then a queue. */ +const LOCK_WARN = 0.2 +const LOCK_CRIT = 0.5 +/** The app's p95 this many times the database's: the time went somewhere else. */ +const APP_GAP = 5 + +function Columns() { + return ( + <> + + + + +