From 0f7a715163fcd3fdddbc7336690e6391f33decf9 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 17:36:18 +0200 Subject: [PATCH 01/21] feat(integrations): Google Cloud connector API and setup script Adds gcp_connectors, GcpConnectorService and /v2/integrations/gcp: an org registers one connector per Google Cloud project and its owner runs the generated gcloud script, which routes the project's logs through Pub/Sub to the ingest gateway and, when the deployment has a Google identity, creates a read-only metrics account Maple may impersonate. --- .env.example | 5 + apps/api/src/resources/env.ts | 1 + .../routes/v2/integrations-gcp.http.test.ts | 269 + .../src/routes/v2/integrations-gcp.http.ts | 91 + apps/api/src/routes/v2/v2-test-support.ts | 4 + apps/api/src/runtime/http-graph.ts | 2 + apps/api/src/runtime/service-graph.ts | 2 + docs/api-v2.md | 11 +- packages/backend/src/platform/Env.ts | 6 + .../src/services/audit/audit-actions.ts | 2 + .../integrations/GcpConnectorService.test.ts | 165 + .../integrations/GcpConnectorService.ts | 215 + .../integrations/gcp/setup-scripts.test.ts | 157 + .../integrations/gcp/setup-scripts.ts | 150 + .../src/services/org/OrganizationService.ts | 3 + .../migration.sql | 17 + .../snapshot.json | 18143 ++++++++++++++++ packages/db/src/schema/gcp-connectors.ts | 28 + packages/db/src/schema/index.ts | 1 + packages/domain/package.json | 1 + packages/domain/src/gcp.test.ts | 26 + packages/domain/src/gcp.ts | 14 + .../anticipated-error-identifiers.ts | 1 + packages/domain/src/http/integrations.ts | 20 +- packages/domain/src/http/v2/api.ts | 2 + packages/domain/src/http/v2/index.ts | 1 + .../domain/src/http/v2/integrations-gcp.ts | 238 + packages/domain/src/http/v2/openapi.test.ts | 4 + packages/domain/src/http/v2/public-id.ts | 1 + packages/primitives/src/index.ts | 10 + 30 files changed, 19585 insertions(+), 5 deletions(-) create mode 100644 apps/api/src/routes/v2/integrations-gcp.http.test.ts create mode 100644 apps/api/src/routes/v2/integrations-gcp.http.ts create mode 100644 packages/backend/src/services/integrations/GcpConnectorService.test.ts create mode 100644 packages/backend/src/services/integrations/GcpConnectorService.ts create mode 100644 packages/backend/src/services/integrations/gcp/setup-scripts.test.ts create mode 100644 packages/backend/src/services/integrations/gcp/setup-scripts.ts create mode 100644 packages/db/drizzle/20261008152420_gcp_connectors/migration.sql create mode 100644 packages/db/drizzle/20261008152420_gcp_connectors/snapshot.json create mode 100644 packages/db/src/schema/gcp-connectors.ts create mode 100644 packages/domain/src/gcp.test.ts create mode 100644 packages/domain/src/gcp.ts create mode 100644 packages/domain/src/http/v2/integrations-gcp.ts diff --git a/.env.example b/.env.example index 655b4e0fc2..fee5836d97 100644 --- a/.env.example +++ b/.env.example @@ -188,6 +188,11 @@ INGEST_REQUIRE_TLS=false # CLOUDFLARE_OAUTH_REVOKE_URL=https://dash.cloudflare.com/oauth2/revoke # MAPLE_CLOUDFLARE_API_BASE_URL=https://api.cloudflare.com/client/v4 +# Google Cloud integration (the project owner runs a generated gcloud script in Cloud Shell). +# Logs need no configuration. For metrics, set the email of a Google service account Maple owns; +# the setup script lets it impersonate a read-only metrics account in the customer's project. +# MAPLE_GCP_SERVICE_ACCOUNT_EMAIL= + # OpenTelemetry (API self-observability) — read by @maple-dev/effect-sdk # MAPLE_ENVIRONMENT=local # "local" = no export; any other value enables OTLP # MAPLE_ENDPOINT=http://127.0.0.1:3474 # Ingest gateway endpoint (enriches with org_id) diff --git a/apps/api/src/resources/env.ts b/apps/api/src/resources/env.ts index e38ccc8cdd..46814eecfb 100644 --- a/apps/api/src/resources/env.ts +++ b/apps/api/src/resources/env.ts @@ -79,4 +79,5 @@ export const apiConfiguredEnv = (stage: MapleStage, region: MapleRegion, domains optionalSecret("GITHUB_APP_WEBHOOK_SECRET"), cloudflareOAuthEnv, planetScaleOAuthEnv, + optionalPlain("MAPLE_GCP_SERVICE_ACCOUNT_EMAIL"), ) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts new file mode 100644 index 0000000000..7484c8d600 --- /dev/null +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -0,0 +1,269 @@ +import { afterEach, describe, expect, it } from "@effect/vitest" +import { ConfigProvider, Context, Effect, Layer, ManagedRuntime, Schema } from "effect" +import { HttpRouter } from "effect/http" +import { HttpApiBuilder } from "effect/http-api" +import { OrgId, UserId } from "@maple/domain/http" +import { MapleApiV2, encodePublicId } from "@maple/domain/http/v2" +import { Env } from "@maple/backend/platform/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { ApiAuthorizationV2Layer } from "@maple/backend/services/auth/ApiAuthorizationV2Layer" +import { AuthService } from "@maple/backend/services/auth/AuthService" +import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" +import { DashboardPersistenceService } from "@maple/backend/services/dashboards/DashboardPersistenceService" +import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService" +import { ApiKeysService } from "@maple/backend/services/org/ApiKeysService" +import { V2TransportErrorBoundaryLive } from "./error-envelope" +import { + AlertsServiceStubLayer, + AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, + ConfigResourceServiceStubsLayer, + PlanetScaleServiceStubsLayer, + TelemetryServiceStubsLayer, +} from "./v2-test-support" + +/** + * `/v2/integrations/gcp` over an embedded PGlite with the real connector service: the wire shape + * a script parses, the admin gate, and who can reach the push secret. + */ + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const ORG = Schema.decodeUnknownSync(OrgId)("org_gcp_e2e") +const USER = Schema.decodeUnknownSync(UserId)("user_gcp_e2e") +const BASE = "/v2/integrations/gcp" + +const testConfig = () => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3478", + MCP_PORT: "3479", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + MAPLE_INGEST_PUBLIC_URL: "https://ingest.test", + INTERNAL_SERVICE_TOKEN: "test-internal-token", + }), + ) + +const makeHarness = () => { + const testDb = createTestDb(createdDbs) + const envLive = Env.layer.pipe(Layer.provide(testConfig())) + const servicesLive = Layer.mergeAll( + ApiKeysService.layer, + AuthService.layer, + DashboardPersistenceService.layer, + SharedDashboardService.layer, + ).pipe(Layer.provideMerge(Layer.mergeAll(envLive, testDb.layer))) + + const routes = HttpApiBuilder.layer(MapleApiV2).pipe( + Layer.provide(AllV2GroupLayersLive), + Layer.provide(V2TransportErrorBoundaryLive), + Layer.provide(AlertsServiceStubLayer), + Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(PlanetScaleServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), + Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layerMemory), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), + Layer.provideMerge(servicesLive), + ) + const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { disableLogger: true }) + const runtime = ManagedRuntime.make(servicesLive) + + /** Root-role key (API keys resolve as `root` unless their metadata pins roles). */ + const adminKey = (scopes?: ReadonlyArray) => + runtime.runPromise( + Effect.gen(function* () { + const service = yield* ApiKeysService + return yield* service.create(ORG, USER, { name: "gcp-admin", scopes }) + }), + ) + + const memberKey = () => + runtime.runPromise( + Effect.gen(function* () { + const service = yield* ApiKeysService + return yield* service.create(ORG, USER, { + name: "gcp-member", + metadataJson: { source: "maple_cli", roles: ["org:member"], deviceName: "laptop" }, + }) + }), + ) + + const request = async (method: string, path: string, token: string, body?: unknown) => { + const response = await handler( + new Request(`http://maple.test${path}`, { + method, + headers: { + authorization: `Bearer ${token}`, + ...(body !== undefined ? { "content-type": "application/json" } : undefined), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }), + Context.empty() as never, + ) + const text = await response.text() + return { status: response.status, body: text.length === 0 ? null : JSON.parse(text) } + } + + return { + request, + adminKey, + memberKey, + dispose: async () => { + await disposeHandler() + await runtime.dispose() + }, + } +} + +describe("v2 gcp integration over HTTP", () => { + it("creates a connector, lists it for any member, and deletes it with a cleanup script", async () => { + const harness = makeHarness() + const admin = await harness.adminKey() + const member = await harness.memberKey() + + const created = await harness.request("POST", `${BASE}/connectors`, admin.secret, { + project_id: "acme-prod", + }) + expect(created.status).toBe(200) + expect(created.body).toEqual({ + id: expect.stringMatching(/^gcpc_/), + object: "gcp_connector", + project_id: "acme-prod", + created_at: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T.*Z$/), + last_log_received_at: null, + last_log_error: null, + }) + + const status = await harness.request("GET", BASE, member.secret) + expect(status.status).toBe(200) + expect(status.body).toEqual({ + object: "gcp_integration", + metrics_available: false, + connectors: [created.body], + }) + + const path = `${BASE}/connectors/${created.body.id}` + const deleted = await harness.request("DELETE", path, admin.secret) + expect(deleted.status).toBe(200) + expect(deleted.body).toEqual({ + id: created.body.id, + object: "gcp_connector", + deleted: true, + cleanup_script: expect.stringContaining("gcloud logging sinks delete"), + }) + + const gone = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) + expect(gone.status).toBe(404) + expect(gone.body.error).toMatchObject({ + _tag: "@maple/http/errors/IntegrationsNotFoundError", + code: "integration_not_found", + }) + await harness.dispose() + }) + + it("renders the scripts for an admin, with the push secret only in the setup script", async () => { + const harness = makeHarness() + const admin = await harness.adminKey() + const created = await harness.request("POST", `${BASE}/connectors`, admin.secret, { + project_id: "acme-prod", + }) + const path = `${BASE}/connectors/${created.body.id}/setup_scripts` + + const { status, body } = await harness.request("POST", path, admin.secret, { + exclude_gke_container_logs: true, + }) + expect(status).toBe(200) + expect(Object.keys(body).sort()).toEqual(["cleanup_script", "object", "setup_script"]) + expect(body.object).toBe("gcp_connector.setup_scripts") + expect(body.setup_script).toMatch( + /PUSH_ENDPOINT='https:\/\/ingest\.test\/v1\/logpush\/gcp\/[0-9a-f-]{36}\?secret=maple_gcp_/, + ) + expect(body.setup_script).toContain("PROJECT_ID='acme-prod'") + expect(body.setup_script).toContain('NOT resource.type="k8s_container"') + expect(body.cleanup_script).not.toContain("maple_gcp_") + + const defaults = await harness.request("POST", path, admin.secret, {}) + expect(defaults.body.setup_script).not.toContain("k8s_container") + await harness.dispose() + }) + + it("keeps the push secret from non-admin members and from read-scoped keys", async () => { + const harness = makeHarness() + const admin = await harness.adminKey() + const member = await harness.memberKey() + const readOnly = await harness.adminKey(["integrations:read"]) + const created = await harness.request("POST", `${BASE}/connectors`, admin.secret, { + project_id: "acme-prod", + }) + const path = `${BASE}/connectors/${created.body.id}` + + for (const [method, target, body] of [ + ["POST", `${BASE}/connectors`, { project_id: "other-project" }], + ["POST", `${path}/setup_scripts`, {}], + ["DELETE", path, undefined], + ] as const) { + const asMember = await harness.request(method, target, member.secret, body) + expect(asMember.status).toBe(403) + expect(asMember.body.error).toMatchObject({ + type: "permission_error", + code: "insufficient_permissions", + }) + const asReadOnly = await harness.request(method, target, readOnly.secret, body) + expect(asReadOnly.status).toBe(403) + expect(asReadOnly.body.error.code).toBe("insufficient_scope") + } + + const status = await harness.request("GET", BASE, readOnly.secret) + expect(status.status).toBe(200) + expect(status.body.connectors).toHaveLength(1) + expect(JSON.stringify(status.body)).not.toContain("maple_gcp_") + await harness.dispose() + }) + + it("answers a project that is already connected with a 409 naming the field", async () => { + const harness = makeHarness() + const admin = await harness.adminKey() + await harness.request("POST", `${BASE}/connectors`, admin.secret, { project_id: "acme-prod" }) + + const { status, body } = await harness.request("POST", `${BASE}/connectors`, admin.secret, { + project_id: "acme-prod", + }) + expect(status).toBe(409) + expect(body.error).toMatchObject({ + _tag: "@maple/http/errors/GcpProjectAlreadyConnectedError", + type: "conflict_error", + code: "gcp_project_already_connected", + param: "project_id", + }) + await harness.dispose() + }) + + it("rejects anything that is not a Google Cloud project id or a gcpc_ id before the service", async () => { + const harness = makeHarness() + const admin = await harness.adminKey() + + for (const projectId of ["acme'; rm -rf /; '", "Acme-Prod", "short", "acme-prod-", "$(id)-project"]) { + const { status, body } = await harness.request("POST", `${BASE}/connectors`, admin.secret, { + project_id: projectId, + }) + expect(status, projectId).toBe(400) + expect(body.error.type).toBe("invalid_request_error") + } + + const wrongPrefix = encodePublicId("scrp", "018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8") + const { status } = await harness.request("DELETE", `${BASE}/connectors/${wrongPrefix}`, admin.secret) + expect(status).toBe(400) + + const empty = await harness.request("GET", BASE, admin.secret) + expect(empty.body.connectors).toEqual([]) + await harness.dispose() + }) +}) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.ts b/apps/api/src/routes/v2/integrations-gcp.http.ts new file mode 100644 index 0000000000..f2de5f42ce --- /dev/null +++ b/apps/api/src/routes/v2/integrations-gcp.http.ts @@ -0,0 +1,91 @@ +import { HttpApiBuilder } from "effect/http-api" +import { CurrentTenant } from "@maple/domain/http" +import type { V2GcpConnector } from "@maple/domain/http/v2" +import { + MapleApiV2, + isoTimestamp, + isoTimestampOrNull, + V2InsufficientPermissions, +} from "@maple/domain/http/v2" +import { Effect } from "effect" +import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService" +import { requireAdmin } from "@maple/backend/services/auth/auth" +import { + GcpConnectorService, + type GcpConnector, +} from "@maple/backend/services/integrations/GcpConnectorService" + +// Reading the status is open to every member. Everything else is admin-only: a connector decides +// which project may write logs into the org, and its setup script carries the push secret. +const adminOnly = () => V2InsufficientPermissions.make("Only org admins can manage Google Cloud connectors") + +const toV2Connector = (connector: GcpConnector): V2GcpConnector => ({ + id: connector.id, + object: "gcp_connector", + project_id: connector.projectId, + created_at: isoTimestamp(connector.createdAt), + last_log_received_at: isoTimestampOrNull(connector.lastLogReceivedAt), + last_log_error: connector.lastLogError, +}) + +export const HttpV2GcpIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "gcpIntegration", (handlers) => + Effect.gen(function* () { + const gcp = yield* GcpConnectorService + + return handlers + .handle("status", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const status = yield* gcp.status(tenant.orgId) + return { + object: "gcp_integration" as const, + metrics_available: status.metricsAvailable, + connectors: status.connectors.map(toV2Connector), + } + }), + ) + .handle("createConnector", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, adminOnly) + const connector = yield* gcp.create(tenant.orgId, tenant.userId, payload.project_id) + yield* recordHttpAudit("gcp_connector.created", { + resourceId: connector.id, + metadata: { project_id: connector.projectId }, + }) + return toV2Connector(connector) + }), + ) + .handle("setupScripts", ({ params, payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, adminOnly) + const scripts = yield* gcp.scripts(tenant.orgId, params.id, { + excludeGkeContainerLogs: payload.exclude_gke_container_logs === true, + }) + return { + object: "gcp_connector.setup_scripts" as const, + setup_script: scripts.setupScript, + cleanup_script: scripts.cleanupScript, + } + }), + ) + .handle("deleteConnector", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, adminOnly) + const deleted = yield* gcp.delete(tenant.orgId, params.id) + yield* recordHttpAudit("gcp_connector.deleted", { + resourceId: params.id, + metadata: { project_id: deleted.projectId }, + }) + return { + id: params.id, + object: "gcp_connector" as const, + deleted: true as const, + cleanup_script: deleted.cleanupScript, + } + }), + ) + }), +) diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index faacb31b4e..f2b6141f99 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -39,7 +39,9 @@ import { HttpV2AttributeMappingsLive } from "./attribute-mappings.http" import { HttpV2DashboardsLive } from "./dashboards.http" import { HttpV2IngestKeysLive } from "./ingest-keys.http" import { HttpV2ChatIntegrationsLive } from "./integrations-chat.http" +import { HttpV2GcpIntegrationsLive } from "./integrations-gcp.http" import { HttpV2PlanetScaleIntegrationsLive } from "./integrations.http" +import { GcpConnectorService } from "@maple/backend/services/integrations/GcpConnectorService" import { HttpV2ErrorIssuesLive } from "./error-issues.http" import { HttpV2AnomaliesLive } from "./anomalies.http" import { HttpV2InvestigationsLive } from "./investigations.http" @@ -152,6 +154,8 @@ const v2GroupLayersExceptOnboardingChecklist = (chatWorkspace: Layer.Layer` shares the `integrations:read` / `integrations:write` family. Two things never move with a provider, no matter how much of it is promoted: the OAuth **callback** and any webhook **receiver**. Both are raw `HttpRouter` routes under `/api/…` serving a browser redirect or a provider POST, and a receiver URL is already registered in the provider's settings. So `POST /v2/integrations/planetscale/connect` still mints a `/api/integrations/planetscale/callback` URL, and `webhook_config` still reports a `/api/…` receiver. That is the intended end state, not leftover v1. @@ -50,7 +52,7 @@ POST /v2/traces/search complex reads are POST .../search Every v2 object has a prefixed public ID (`key_4CzLmR…`, `dash_…`, `alrt_…`). Public IDs are opaque; internally they are a reversible base58 encoding of the internal ID, computed at the API boundary in `packages/domain/src/http/v2/public-id.ts`, which also holds the prefix registry (`PublicIdPrefixes`, the source of truth). No database migration: rows keep their raw UUIDs / internal strings. -Prefixes: `key` (API key), `ingk` (ingest key), `dash` (dashboard), `dbv` (dashboard version), `dshr` (dashboard share), `dtpl` (dashboard template), `alrt` (alert rule), `dest` (alert destination), `inc` (alert incident), `evt` (alert delivery event), `actor` (actor), `alog` (audit log entry), `einc` (error incident), `iss` (error issue), `inv` (investigation), `anom` (anomaly incident), `scrp` (scrape target), `rec` (recommendation), `amap` (attribute mapping), `srep` (session replay), `mdev` (mobile device), `chatw` (chat workspace), `afb` (agent feedback), and `log` (synthetic log identity); `we` is reserved for webhooks. +Prefixes: `key` (API key), `ingk` (ingest key), `dash` (dashboard), `dbv` (dashboard version), `dshr` (dashboard share), `dtpl` (dashboard template), `alrt` (alert rule), `dest` (alert destination), `inc` (alert incident), `evt` (alert delivery event), `actor` (actor), `alog` (audit log entry), `einc` (error incident), `iss` (error issue), `inv` (investigation), `anom` (anomaly incident), `scrp` (scrape target), `rec` (recommendation), `amap` (attribute mapping), `srep` (session replay), `mdev` (mobile device), `chatw` (chat workspace), `afb` (agent feedback), `gcpc` (Google Cloud connector), and `log` (synthetic log identity); `we` is reserved for webhooks. Exception: Clerk-issued `org_…` / `user_…` IDs are already prefixed public IDs and pass through unchanged. @@ -172,6 +174,7 @@ Implemented in phases; the pilot (`api_keys`) ships first and proves every conve | `attribute_mappings` ✅ | CRUD | `IngestAttributeMappingService` | | `integrations/chat_*` ✅ | connector list (manifest + linked workspaces) + admin-only install/settings/unlink; the chat platform is a value, never a path segment | `ChatWorkspaceService` | | `integrations/planetscale` ✅ | status + connect/organizations/`select_organization`/`metrics_token`/disconnect + databases/`webhook_config`/`query_insights`/`events` | `PlanetScaleConnectionService`, `PlanetScaleOAuthService`, `PlanetScaleService` | +| `integrations/gcp` ✅ | status (connectors + `metrics_available`) + admin-only connector create/delete and `setup_scripts` | `GcpConnectorService` | | `session_replays` ✅ | `search`/retrieve + events/transcript/`for_trace` (reduced; `facets`/`trace-summaries` deferred) | `sessionReplays` | | `mobile_devices` ✅ | list (mine) / `PUT …/{token}` register-or-refresh / `DELETE …/{token}`; per user, per org; drives push fan-out | `MobileDevicesService` | | `agent_feedback` ✅ | create/list; feedback about Maple from agents: `kind`, `agent` (type/name/model/version), `reason`, optional `impact`/`details`/`related_to` | `AgentFeedbackService` | diff --git a/packages/backend/src/platform/Env.ts b/packages/backend/src/platform/Env.ts index e4faa76425..e68e2847b5 100644 --- a/packages/backend/src/platform/Env.ts +++ b/packages/backend/src/platform/Env.ts @@ -149,6 +149,11 @@ export interface EnvConfig { readonly GITHUB_APP_CLIENT_SECRET: Option.Option> readonly GITHUB_APP_WEBHOOK_SECRET: Option.Option> readonly GITHUB_API_BASE_URL: string + /** + * Maple's own Google service account. Customers grant it permission to impersonate the + * metrics reader their setup script creates. Unset: the GCP integration is logs-only. + */ + readonly MAPLE_GCP_SERVICE_ACCOUNT_EMAIL: Option.Option readonly CLOUDFLARE_OAUTH_CLIENT_ID: Option.Option readonly CLOUDFLARE_OAUTH_CLIENT_SECRET: Option.Option> readonly CLOUDFLARE_OAUTH_SCOPES: string @@ -256,6 +261,7 @@ const envConfig = Config.all({ GITHUB_APP_CLIENT_SECRET: optionalRedacted("GITHUB_APP_CLIENT_SECRET"), GITHUB_APP_WEBHOOK_SECRET: optionalRedacted("GITHUB_APP_WEBHOOK_SECRET"), GITHUB_API_BASE_URL: stringWithDefault("GITHUB_API_BASE_URL", "https://api.github.com"), + MAPLE_GCP_SERVICE_ACCOUNT_EMAIL: optionalString("MAPLE_GCP_SERVICE_ACCOUNT_EMAIL"), CLOUDFLARE_OAUTH_CLIENT_ID: optionalString("CLOUDFLARE_OAUTH_CLIENT_ID"), CLOUDFLARE_OAUTH_CLIENT_SECRET: optionalRedacted("CLOUDFLARE_OAUTH_CLIENT_SECRET"), // Cloudflare OAuth scope ids are DOT-delimited (mirroring API-token permission names; diff --git a/packages/backend/src/services/audit/audit-actions.ts b/packages/backend/src/services/audit/audit-actions.ts index 09bd7860e0..dcde1efaf7 100644 --- a/packages/backend/src/services/audit/audit-actions.ts +++ b/packages/backend/src/services/audit/audit-actions.ts @@ -61,6 +61,8 @@ export const AuditResources = { dashboard_share: { prefix: PublicIdPrefixes.dashboardShare, verbs: ["created", "rotated", "deleted"] }, /** Verbs mirror the issue event types — `recordEvent` audits every one it attributes. */ error_issue: { prefix: PublicIdPrefixes.errorIssue, verbs: ErrorIssueEventType.literals }, + /** One connected Google Cloud project; the project id is in `metadata`. */ + gcp_connector: { prefix: PublicIdPrefixes.gcpConnector, verbs: ["created", "deleted"] }, /** Org-singleton public/private pair; which one rolled is in `metadata`. */ ingest_key: { verbs: ["rolled"] }, /** diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts new file mode 100644 index 0000000000..6086e13662 --- /dev/null +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -0,0 +1,165 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { hashIngestKey } from "@maple/db" +import { gcpConnectorResourceNames } from "@maple/domain/gcp" +import { GcpProjectId, OrgId, UserId } from "@maple/domain/primitives" +import { Env } from "@maple/backend/platform/Env" +import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/platform/test-pglite" +import { GcpConnectorService, type GcpConnector } from "./GcpConnectorService" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const LOOKUP_HMAC_KEY = "maple-test-lookup-secret" +const MAPLE_ACCOUNT = "collector@maple-prod.iam.gserviceaccount.com" + +// A fresh layer per call: one test builds the service twice, under two configurations. +const makeLayer = (testDb: TestDb, mapleServiceAccount?: string) => + Layer.effect(GcpConnectorService, GcpConnectorService.make).pipe( + Layer.provide(testDb.layer), + Layer.provide(Env.layer), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 9).toString("base64"), + // Padded: the gateway trims this key before hashing, so the API must too. + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: ` ${LOOKUP_HMAC_KEY} `, + MAPLE_INGEST_PUBLIC_URL: "https://ingest.test/", + ...(mapleServiceAccount === undefined + ? undefined + : { MAPLE_GCP_SERVICE_ACCOUNT_EMAIL: mapleServiceAccount }), + }), + ), + ), + ) + +const orgId = Schema.decodeUnknownSync(OrgId)("org_gcp") +const otherOrgId = Schema.decodeUnknownSync(OrgId)("org_other") +const userId = Schema.decodeUnknownSync(UserId)("user_1") +const projectId = Schema.decodeUnknownSync(GcpProjectId) + +interface StoredConnector { + readonly secret_ciphertext: string + readonly secret_hash: string + readonly created_by: string +} + +describe("GcpConnectorService", () => { + it.effect("stores a connector whose secret hash is the one the ingest gateway computes", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + const connector = yield* gcp.create(orgId, userId, projectId("acme-prod")) + assert.strictEqual(connector.projectId, "acme-prod") + assert.isNull(connector.lastLogReceivedAt) + assert.isNull(connector.lastLogError) + + const { setupScript } = yield* gcp.scripts(orgId, connector.id, { + excludeGkeContainerLogs: false, + }) + const endpoint = /PUSH_ENDPOINT='([^']+)'/.exec(setupScript)?.[1] + assert.isDefined(endpoint) + const url = new URL(endpoint!) + assert.strictEqual( + url.origin + url.pathname, + `https://ingest.test/v1/logpush/gcp/${connector.id}`, + ) + const secret = url.searchParams.get("secret")! + assert.match(secret, /^maple_gcp_[A-Za-z0-9_-]{32}$/) + + const row = yield* Effect.promise(() => + queryFirstRow(testDb, "SELECT * FROM gcp_connectors WHERE id = $1", [ + connector.id, + ]), + ) + assert.strictEqual(row!.secret_hash, hashIngestKey(secret, LOOKUP_HMAC_KEY)) + assert.notInclude(row!.secret_ciphertext, secret) + assert.strictEqual(row!.created_by, userId) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("refuses a project the organization already connected, but not another organization's", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + yield* gcp.create(orgId, userId, projectId("acme-prod")) + const duplicate = yield* Effect.flip(gcp.create(orgId, userId, projectId("acme-prod"))) + assert.strictEqual(duplicate._tag, "@maple/http/errors/GcpProjectAlreadyConnectedError") + + // Each org gets its own connector, and with it its own resource names in the project. + const theirs = yield* gcp.create(otherOrgId, userId, projectId("acme-prod")) + const ours = (yield* gcp.status(orgId)).connectors + assert.lengthOf(ours, 1) + assert.notDeepEqual(gcpConnectorResourceNames(theirs.id), gcpConnectorResourceNames(ours[0]!.id)) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("reports the gateway's last push and renders metrics steps only with a Google identity", () => { + const testDb = createTestDb(trackedDbs) + const withoutMetrics = Effect.gen(function* () { + const gcp = yield* GcpConnectorService + const connector = yield* gcp.create(orgId, userId, projectId("acme-prod")) + // What the ingest gateway writes after a rejected push. + yield* Effect.promise(() => + testDb.pglite.query( + "UPDATE gcp_connectors SET last_received_at = $1, last_error = $2 WHERE id = $3", + ["2026-10-08T09:12:00.000Z", "payload was not a LogEntry", connector.id], + ), + ) + const status = yield* gcp.status(orgId) + assert.isFalse(status.metricsAvailable) + assert.deepStrictEqual(status.connectors, [ + { + ...connector, + lastLogReceivedAt: Date.parse("2026-10-08T09:12:00.000Z"), + lastLogError: "payload was not a LogEntry", + }, + ]) + const scripts = yield* gcp.scripts(orgId, connector.id, { excludeGkeContainerLogs: true }) + assert.notInclude(scripts.setupScript, "service-accounts") + assert.include(scripts.setupScript, 'NOT resource.type="k8s_container"') + return connector + }).pipe(Effect.provide(makeLayer(testDb))) + + const withMetrics = Effect.fnUntraced(function* (connectorId: GcpConnector["id"]) { + const gcp = yield* GcpConnectorService + assert.isTrue((yield* gcp.status(orgId)).metricsAvailable) + const scripts = yield* gcp.scripts(orgId, connectorId, { excludeGkeContainerLogs: false }) + assert.include(scripts.setupScript, `MAPLE_SERVICE_ACCOUNT='${MAPLE_ACCOUNT}'`) + assert.notInclude(scripts.cleanupScript, "maple_gcp_") + }) + + return withoutMetrics.pipe( + Effect.flatMap((connector) => + withMetrics(connector.id).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))), + ), + ) + }) + + it.effect("scopes scripts and deletion to the owning organization", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + const connector = yield* gcp.create(orgId, userId, projectId("acme-prod")) + + const options = { excludeGkeContainerLogs: false } + const foreignScripts = yield* Effect.flip(gcp.scripts(otherOrgId, connector.id, options)) + assert.strictEqual(foreignScripts._tag, "@maple/http/errors/IntegrationsNotFoundError") + const foreignDelete = yield* Effect.flip(gcp.delete(otherOrgId, connector.id)) + assert.strictEqual(foreignDelete._tag, "@maple/http/errors/IntegrationsNotFoundError") + + const deleted = yield* gcp.delete(orgId, connector.id) + assert.strictEqual(deleted.projectId, "acme-prod") + assert.include(deleted.cleanupScript, gcpConnectorResourceNames(connector.id).sink) + assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, []) + const again = yield* Effect.flip(gcp.delete(orgId, connector.id)) + assert.strictEqual(again._tag, "@maple/http/errors/IntegrationsNotFoundError") + }).pipe(Effect.provide(makeLayer(testDb))) + }) +}) diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts new file mode 100644 index 0000000000..bffbcbae6e --- /dev/null +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -0,0 +1,215 @@ +import { randomBytes, randomUUID } from "node:crypto" +import { gcpConnectors, hashIngestKey, parseIngestKeyLookupHmacKey, type GcpConnectorRow } from "@maple/db" +import { + GcpProjectAlreadyConnectedError, + IntegrationsNotFoundError, + IntegrationsPersistenceError, +} from "@maple/domain/http" +import { GcpConnectorId, type GcpProjectId, type OrgId, type UserId } from "@maple/domain/primitives" +import { and, asc, eq } from "drizzle-orm" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { decryptAes256Gcm, encryptAes256Gcm, parseBase64Aes256GcmKey } from "@maple/backend/platform/Crypto" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { Env } from "@maple/backend/platform/Env" +import { dateToMs, msToDate } from "@maple/backend/platform/time" +import { renderGcpCleanupScript, renderGcpSetupScript } from "./gcp/setup-scripts" + +export interface GcpConnector { + readonly id: GcpConnectorId + readonly projectId: string + readonly createdAt: number + /** Last log push the ingest gateway accepted from this project. Null until the first one. */ + readonly lastLogReceivedAt: number | null + readonly lastLogError: string | null +} + +export interface GcpConnectorServiceApi { + readonly status: (orgId: OrgId) => Effect.Effect< + { + /** False when this deployment has no Google identity, so connectors are logs-only. */ + readonly metricsAvailable: boolean + readonly connectors: ReadonlyArray + }, + IntegrationsPersistenceError + > + readonly create: ( + orgId: OrgId, + userId: UserId, + projectId: GcpProjectId, + ) => Effect.Effect + readonly scripts: ( + orgId: OrgId, + connectorId: GcpConnectorId, + options: { readonly excludeGkeContainerLogs: boolean }, + ) => Effect.Effect< + { readonly setupScript: string; readonly cleanupScript: string }, + IntegrationsNotFoundError | IntegrationsPersistenceError + > + /** Deleting the row is the disconnect: the ingest gateway stops accepting the project's pushes. */ + readonly delete: ( + orgId: OrgId, + connectorId: GcpConnectorId, + ) => Effect.Effect< + { readonly projectId: string; readonly cleanupScript: string }, + IntegrationsNotFoundError | IntegrationsPersistenceError + > +} + +const toPersistenceError = makePersistenceErrorMapper( + IntegrationsPersistenceError, + "Google Cloud connector database error", +) + +const decodeConnectorId = Schema.decodeUnknownSync(GcpConnectorId) + +// Binds the ciphertext to its row, so a stored secret cannot be moved onto another connector. +const secretAad = (connectorId: string) => Buffer.from(`gcp_connectors:v1:${connectorId}`, "utf8") + +const toConnector = (row: GcpConnectorRow): GcpConnector => ({ + id: decodeConnectorId(row.id), + projectId: row.projectId, + createdAt: dateToMs(row.createdAt), + lastLogReceivedAt: dateToMs(row.lastReceivedAt), + lastLogError: row.lastError, +}) + +const notFound = () => new IntegrationsNotFoundError({ message: "No such Google Cloud connector." }) + +export class GcpConnectorService extends Context.Service()( + "@maple/api/services/GcpConnectorService", + { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const encryptionKey = yield* parseBase64Aes256GcmKey( + Redacted.value(env.MAPLE_INGEST_KEY_ENCRYPTION_KEY), + (message) => new IntegrationsPersistenceError({ message }), + ) + // The ingest gateway hashes a pushed secret with this key and looks the row up by it. + const lookupHmacKey = yield* Effect.try({ + try: () => parseIngestKeyLookupHmacKey(Redacted.value(env.MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY)), + catch: () => + new IntegrationsPersistenceError({ + message: "MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY is required", + }), + }) + const ingestBaseUrl = env.MAPLE_INGEST_PUBLIC_URL.replace(/\/+$/, "") + const mapleServiceAccountEmail = Option.getOrUndefined(env.MAPLE_GCP_SERVICE_ACCOUNT_EMAIL) + const dbExecute = makeDbExecute(database, "GcpConnectorService", toPersistenceError) + + const status = Effect.fn("GcpConnectorService.status")(function* (orgId: OrgId) { + const rows = yield* dbExecute((db) => + db + .select() + .from(gcpConnectors) + .where(eq(gcpConnectors.orgId, orgId)) + .orderBy(asc(gcpConnectors.createdAt), asc(gcpConnectors.id)), + ) + return { + metricsAvailable: mapleServiceAccountEmail !== undefined, + connectors: rows.map(toConnector), + } + }) + + const create = Effect.fn("GcpConnectorService.create")(function* ( + orgId: OrgId, + userId: UserId, + projectId: GcpProjectId, + ) { + const id = randomUUID() + const secret = `maple_gcp_${randomBytes(24).toString("base64url")}` + const encrypted = yield* encryptAes256Gcm( + secret, + encryptionKey, + (message) => new IntegrationsPersistenceError({ message }), + secretAad(id), + ) + const now = msToDate(yield* Clock.currentTimeMillis) + const rows = yield* dbExecute((db) => + db + .insert(gcpConnectors) + .values({ + id, + orgId, + projectId, + secretCiphertext: encrypted.ciphertext, + secretIv: encrypted.iv, + secretTag: encrypted.tag, + secretHash: hashIngestKey(secret, lookupHmacKey), + createdBy: userId, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: [gcpConnectors.orgId, gcpConnectors.projectId] }) + .returning(), + ) + const row = rows[0] + if (row === undefined) { + return yield* new GcpProjectAlreadyConnectedError({ + projectId, + message: `Google Cloud project ${projectId} is already connected.`, + }) + } + return toConnector(row) + }) + + const scripts = Effect.fn("GcpConnectorService.scripts")(function* ( + orgId: OrgId, + connectorId: GcpConnectorId, + options: { readonly excludeGkeContainerLogs: boolean }, + ) { + const rows = yield* dbExecute((db) => + db + .select() + .from(gcpConnectors) + .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) + .limit(1), + ) + const row = rows[0] + if (row === undefined) return yield* notFound() + const secret = yield* decryptAes256Gcm( + { ciphertext: row.secretCiphertext, iv: row.secretIv, tag: row.secretTag }, + encryptionKey, + () => + new IntegrationsPersistenceError({ + message: "Failed to decrypt the Google Cloud connector secret", + }), + secretAad(row.id), + ) + return { + setupScript: renderGcpSetupScript({ + connectorId, + projectId: row.projectId, + pushEndpoint: `${ingestBaseUrl}/v1/logpush/gcp/${connectorId}?secret=${secret}`, + mapleServiceAccountEmail, + excludeGkeContainerLogs: options.excludeGkeContainerLogs, + }), + cleanupScript: renderGcpCleanupScript(connectorId, row.projectId), + } + }) + + const deleteConnector = Effect.fn("GcpConnectorService.delete")(function* ( + orgId: OrgId, + connectorId: GcpConnectorId, + ) { + const rows = yield* dbExecute((db) => + db + .delete(gcpConnectors) + .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) + .returning({ projectId: gcpConnectors.projectId }), + ) + const row = rows[0] + if (row === undefined) return yield* notFound() + return { + projectId: row.projectId, + cleanupScript: renderGcpCleanupScript(connectorId, row.projectId), + } + }) + + return { status, create, scripts, delete: deleteConnector } satisfies GcpConnectorServiceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts new file mode 100644 index 0000000000..259650e317 --- /dev/null +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -0,0 +1,157 @@ +import { spawnSync } from "node:child_process" +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { Schema } from "effect" +import { GcpConnectorId } from "@maple/domain/primitives" +import { + gcpLogFilter, + renderGcpCleanupScript, + renderGcpSetupScript, + type GcpSetupScriptInput, +} from "./setup-scripts" + +const connectorId = Schema.decodeUnknownSync(GcpConnectorId)("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8") +const NAME = "maple-018f2b3c4d5e4f708192a3b4" +const SECRET = "maple_gcp_s3cr3t-value" +const WRITER = "serviceAccount:service-1@gcp-sa-logging.iam.gserviceaccount.com" +const MAPLE_ACCOUNT = "collector@maple-prod.iam.gserviceaccount.com" + +const input: GcpSetupScriptInput = { + connectorId, + projectId: "acme-prod", + pushEndpoint: `https://ingest.test/v1/logpush/gcp/${connectorId}?secret=${SECRET}`, + mapleServiceAccountEmail: MAPLE_ACCOUNT, + excludeGkeContainerLogs: false, +} + +// A `gcloud` that records its arguments. `describe` reports "not found" unless the test says +// the resources exist, and the sink's writer identity is the one value a script reads back. +const FAKE_GCLOUD = `#!/usr/bin/env bash +echo "$*" >> "$GCLOUD_LOG" +case "$*" in + *"sinks describe"*"--format"*) echo "${WRITER}" ;; + *" describe "*) [ "$GCLOUD_EXISTS" = 1 ] || exit 1 ;; + *" delete "*) exit "$GCLOUD_DELETE_STATUS" ;; +esac +` + +const tempDirs: string[] = [] +afterEach(() => { + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }) +}) + +/** Run a rendered script in bash against the fake gcloud; returns the commands it issued. */ +const run = (script: string, options: { exists?: boolean; deleteStatus?: number } = {}) => { + const dir = mkdtempSync(join(tmpdir(), "maple-gcp-script-")) + tempDirs.push(dir) + writeFileSync(join(dir, "gcloud"), FAKE_GCLOUD) + chmodSync(join(dir, "gcloud"), 0o755) + const log = join(dir, "gcloud.log") + const result = spawnSync("bash", [], { + input: script, + cwd: dir, + encoding: "utf8", + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + GCLOUD_LOG: log, + GCLOUD_EXISTS: options.exists === true ? "1" : "0", + GCLOUD_DELETE_STATUS: String(options.deleteStatus ?? 0), + }, + }) + return { + dir, + status: result.status, + stdout: result.stdout, + commands: existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : [], + } +} + +describe("renderGcpSetupScript", () => { + it("creates the log pipeline and the metrics account in a fresh project", () => { + const { status, stdout, commands } = run(renderGcpSetupScript(input)) + expect(status).toBe(0) + expect(stdout).toContain("Maple setup complete for acme-prod") + expect(commands).toEqual([ + "services enable pubsub.googleapis.com logging.googleapis.com monitoring.googleapis.com iam.googleapis.com iamcredentials.googleapis.com --project=acme-prod", + `pubsub topics describe ${NAME} --project=acme-prod`, + `pubsub topics create ${NAME} --project=acme-prod`, + `pubsub subscriptions describe ${NAME} --project=acme-prod`, + `pubsub subscriptions create ${NAME} --topic=${NAME} --project=acme-prod --push-endpoint=${input.pushEndpoint} --push-no-wrapper --ack-deadline=30 --message-retention-duration=1d --expiration-period=never --min-retry-delay=10s --max-retry-delay=600s`, + `logging sinks describe ${NAME} --project=acme-prod`, + `logging sinks create ${NAME} pubsub.googleapis.com/projects/acme-prod/topics/${NAME} --log-filter=${gcpLogFilter(false)} --project=acme-prod`, + `logging sinks describe ${NAME} --project=acme-prod --format=value(writerIdentity)`, + `pubsub topics add-iam-policy-binding ${NAME} --project=acme-prod --member=${WRITER} --role=roles/pubsub.publisher`, + `iam service-accounts describe ${NAME}@acme-prod.iam.gserviceaccount.com --project=acme-prod`, + `iam service-accounts create ${NAME} --project=acme-prod --display-name=Maple metrics reader`, + `projects add-iam-policy-binding acme-prod --member=serviceAccount:${NAME}@acme-prod.iam.gserviceaccount.com --role=roles/monitoring.viewer --condition=None`, + `iam service-accounts add-iam-policy-binding ${NAME}@acme-prod.iam.gserviceaccount.com --project=acme-prod --member=serviceAccount:${MAPLE_ACCOUNT} --role=roles/iam.serviceAccountTokenCreator`, + ]) + }) + + it("updates instead of creating when re-run, so an edited filter is applied", () => { + const { status, commands } = run(renderGcpSetupScript(input), { exists: true }) + expect(status).toBe(0) + expect(commands.filter((command) => command.includes(" create "))).toEqual([]) + expect(commands).toContainEqual( + expect.stringMatching(/^pubsub subscriptions update .*--push-no-wrapper/), + ) + expect(commands).toContainEqual( + expect.stringMatching(/^logging sinks update .*--log-filter=NOT log_id/), + ) + }) + + it("omits every metrics step when the deployment has no Google identity", () => { + const script = renderGcpSetupScript({ ...input, mapleServiceAccountEmail: undefined }) + expect(script).not.toContain("service-accounts") + expect(script).not.toContain("monitoring") + const { status, commands } = run(script) + expect(status).toBe(0) + expect(commands[0]).toBe( + "services enable pubsub.googleapis.com logging.googleapis.com --project=acme-prod", + ) + expect(commands.at(-1)).toContain("pubsub topics add-iam-policy-binding") + }) + + it("excludes data-access audit logs and health checks by default, GKE containers on request", () => { + expect(gcpLogFilter(false)).toBe( + 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT log_id("compute.googleapis.com/healthchecks") AND NOT httpRequest.userAgent:"GoogleHC"', + ) + expect(gcpLogFilter(true)).toBe(`${gcpLogFilter(false)} AND NOT resource.type="k8s_container"`) + expect(renderGcpSetupScript({ ...input, excludeGkeContainerLogs: true })).toContain( + `LOG_FILTER='${gcpLogFilter(true)}'`, + ) + }) + + it("passes interpolated values to gcloud verbatim, whatever they contain", () => { + const hostile = `x'; touch pwned; echo '$(touch pwned)` + const { dir, status, commands } = run(renderGcpSetupScript({ ...input, projectId: hostile })) + expect(status).toBe(0) + expect(commands[0]).toContain(`--project=${hostile}`) + expect(existsSync(join(dir, "pwned"))).toBe(false) + }) +}) + +describe("renderGcpCleanupScript", () => { + const script = renderGcpCleanupScript(connectorId, "acme-prod") + + it("carries the secret in the setup script only", () => { + expect(renderGcpSetupScript(input)).toContain(SECRET) + expect(script).not.toContain("secret") + expect(script).not.toContain("maple_gcp_") + }) + + it("removes every resource, continuing past ones that are already gone", () => { + const { status, commands } = run(script, { deleteStatus: 1 }) + expect(status).toBe(0) + expect(commands).toEqual([ + `logging sinks delete ${NAME} --project=acme-prod --quiet`, + `pubsub subscriptions delete ${NAME} --project=acme-prod`, + `pubsub topics delete ${NAME} --project=acme-prod`, + `projects remove-iam-policy-binding acme-prod --member=serviceAccount:${NAME}@acme-prod.iam.gserviceaccount.com --role=roles/monitoring.viewer --condition=None`, + `iam service-accounts delete ${NAME}@acme-prod.iam.gserviceaccount.com --project=acme-prod --quiet`, + ]) + }) +}) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts new file mode 100644 index 0000000000..d518fc9465 --- /dev/null +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -0,0 +1,150 @@ +import { gcpConnectorResourceNames } from "@maple/domain/gcp" +import type { GcpConnectorId } from "@maple/domain/primitives" + +/** Single-quote a value for bash. Every value that reaches a script goes through here. */ +const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` + +// Data Access audit logs record every API read, and health-check probes repeat every few +// seconds per backend: both are high volume and say little about the workload. +const DEFAULT_LOG_FILTER = [ + 'NOT log_id("cloudaudit.googleapis.com/data_access")', + 'NOT log_id("compute.googleapis.com/healthchecks")', + 'NOT httpRequest.userAgent:"GoogleHC"', +] + +export const gcpLogFilter = (excludeGkeContainerLogs: boolean): string => + [...DEFAULT_LOG_FILTER, ...(excludeGkeContainerLogs ? ['NOT resource.type="k8s_container"'] : [])].join( + " AND ", + ) + +const logVariables = (connectorId: GcpConnectorId, projectId: string): string => { + const names = gcpConnectorResourceNames(connectorId) + return `PROJECT_ID=${sh(projectId)} +TOPIC=${sh(names.topic)} +SUBSCRIPTION=${sh(names.subscription)} +SINK=${sh(names.sink)}` +} + +const serviceAccountVariables = (connectorId: GcpConnectorId): string => + `SERVICE_ACCOUNT=${sh(gcpConnectorResourceNames(connectorId).serviceAccountId)} +SERVICE_ACCOUNT_EMAIL="$SERVICE_ACCOUNT@$PROJECT_ID.iam.gserviceaccount.com"` + +const metricsSteps = (connectorId: GcpConnectorId, mapleServiceAccountEmail: string): string => ` +# Metrics: a read-only service account in this project that Maple may impersonate. +${serviceAccountVariables(connectorId)} +MAPLE_SERVICE_ACCOUNT=${sh(mapleServiceAccountEmail)} + +if ! gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud iam service-accounts create "$SERVICE_ACCOUNT" --project="$PROJECT_ID" \\ + --display-name="Maple metrics reader" +fi + +# A new service account can take a minute to become visible to IAM. +retry() { + for _ in 1 2 3 4 5 6; do + "$@" && return + sleep 10 + done + "$@" +} + +retry gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ + --member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" --role=roles/monitoring.viewer \\ + --condition=None >/dev/null + +retry gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" \\ + --member="serviceAccount:$MAPLE_SERVICE_ACCOUNT" --role=roles/iam.serviceAccountTokenCreator >/dev/null +` + +export interface GcpSetupScriptInput { + readonly connectorId: GcpConnectorId + readonly projectId: string + /** The ingest gateway's receiver URL for this connector, including its secret. */ + readonly pushEndpoint: string + /** Maple's own Google service account. Undefined renders a logs-only script. */ + readonly mapleServiceAccountEmail: string | undefined + readonly excludeGkeContainerLogs: boolean +} + +/** + * The script a project owner runs in Cloud Shell: a Log Router sink into a Pub/Sub topic whose + * push subscription delivers each LogEntry to Maple. Every step first looks for the resource, so + * re-running applies an edited filter instead of failing on what already exists. + */ +export const renderGcpSetupScript = (input: GcpSetupScriptInput): string => { + const mapleAccount = input.mapleServiceAccountEmail + const metrics = mapleAccount !== undefined + return `#!/usr/bin/env bash +# Maple: forward this Google Cloud project's logs${metrics ? " and metrics" : ""} to Maple. +# Run in Cloud Shell as a project owner. Safe to re-run. +# Keep this script private: PUSH_ENDPOINT contains this connector's secret. +set -euo pipefail + +# ---- Edit this to change which logs are forwarded, then re-run. ---- +# Cloud Logging query language: https://cloud.google.com/logging/docs/view/logging-query-language +LOG_FILTER=${sh(gcpLogFilter(input.excludeGkeContainerLogs))} +# -------------------------------------------------------------------- + +${logVariables(input.connectorId, input.projectId)} +PUSH_ENDPOINT=${sh(input.pushEndpoint)} +DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" + +gcloud services enable pubsub.googleapis.com logging.googleapis.com${ + metrics ? " monitoring.googleapis.com iam.googleapis.com iamcredentials.googleapis.com" : "" + } \\ + --project="$PROJECT_ID" + +if ! gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud pubsub topics create "$TOPIC" --project="$PROJECT_ID" +fi + +# One LogEntry per request, unwrapped. A day of retention bounds what is replayed after an +# outage, and the subscription never expires while it is idle. +SUBSCRIPTION_FLAGS=( + --project="$PROJECT_ID" + --push-endpoint="$PUSH_ENDPOINT" --push-no-wrapper + --ack-deadline=30 --message-retention-duration=1d --expiration-period=never + --min-retry-delay=10s --max-retry-delay=600s +) +if gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud pubsub subscriptions update "$SUBSCRIPTION" "\${SUBSCRIPTION_FLAGS[@]}" +else + gcloud pubsub subscriptions create "$SUBSCRIPTION" --topic="$TOPIC" "\${SUBSCRIPTION_FLAGS[@]}" +fi + +if gcloud logging sinks describe "$SINK" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud logging sinks update "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" --project="$PROJECT_ID" +else + gcloud logging sinks create "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" --project="$PROJECT_ID" +fi + +# The sink writes as its own Google-managed identity, which needs to publish to the topic. +WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" --project="$PROJECT_ID" --format='value(writerIdentity)')" +gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ + --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher >/dev/null +${mapleAccount === undefined ? "" : metricsSteps(input.connectorId, mapleAccount)} +echo "Maple setup complete for $PROJECT_ID. Logs start arriving in Maple within a few minutes." +` +} + +/** Removes what the setup script created. Carries no secret, so it outlives the connector. */ +export const renderGcpCleanupScript = (connectorId: GcpConnectorId, projectId: string): string => + `#!/usr/bin/env bash +# Maple: remove what the Maple setup script created in this Google Cloud project. +# Run in Cloud Shell as a project owner. No "set -e": every step runs, and a +# NOT_FOUND error only means that resource is already gone. +set -uo pipefail + +${logVariables(connectorId, projectId)} +${serviceAccountVariables(connectorId)} + +gcloud logging sinks delete "$SINK" --project="$PROJECT_ID" --quiet +gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" +gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" +gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ + --member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" --role=roles/monitoring.viewer \\ + --condition=None >/dev/null +gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet + +echo "Maple cleanup finished for $PROJECT_ID." +` diff --git a/packages/backend/src/services/org/OrganizationService.ts b/packages/backend/src/services/org/OrganizationService.ts index 89cd7bc8f5..f9f26f4db8 100644 --- a/packages/backend/src/services/org/OrganizationService.ts +++ b/packages/backend/src/services/org/OrganizationService.ts @@ -32,6 +32,7 @@ import { chatWorkspaces, cliDeviceAuthorizations, cloudflareLogpushConnectors, + gcpConnectors, dashboards, dashboardShares, dashboardVersions, @@ -113,6 +114,8 @@ const ORG_SCOPED_TABLES = [ chatIdentities, digestSubscriptions, cloudflareLogpushConnectors, + // A surviving push secret would keep the ingest gateway accepting logs for a deleted org. + gcpConnectors, errorIssueEvents, errorIssueStates, errorIncidents, diff --git a/packages/db/drizzle/20261008152420_gcp_connectors/migration.sql b/packages/db/drizzle/20261008152420_gcp_connectors/migration.sql new file mode 100644 index 0000000000..56d4e5a5a0 --- /dev/null +++ b/packages/db/drizzle/20261008152420_gcp_connectors/migration.sql @@ -0,0 +1,17 @@ +CREATE TABLE "gcp_connectors" ( + "id" text PRIMARY KEY, + "org_id" text NOT NULL, + "project_id" text NOT NULL, + "secret_ciphertext" text NOT NULL, + "secret_iv" text NOT NULL, + "secret_tag" text NOT NULL, + "secret_hash" text NOT NULL, + "last_received_at" timestamp with time zone, + "last_error" text, + "created_by" text NOT NULL, + "created_at" timestamp with time zone NOT NULL, + "updated_at" timestamp with time zone NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX "gcp_connectors_org_project_idx" ON "gcp_connectors" ("org_id","project_id");--> statement-breakpoint +CREATE UNIQUE INDEX "gcp_connectors_secret_hash_unique" ON "gcp_connectors" ("secret_hash"); \ No newline at end of file diff --git a/packages/db/drizzle/20261008152420_gcp_connectors/snapshot.json b/packages/db/drizzle/20261008152420_gcp_connectors/snapshot.json new file mode 100644 index 0000000000..7486106312 --- /dev/null +++ b/packages/db/drizzle/20261008152420_gcp_connectors/snapshot.json @@ -0,0 +1,18143 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "cede4985-423f-4e80-bdb7-25d4319189ed", + "prevIds": [ + "a9ce02f8-139d-4eb9-bf09-315c5936b0fa" + ], + "ddl": [ + { + "isRlsEnabled": false, + "name": "agent_feedback", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "ai_triage_settings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "alert_delivery_events", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "alert_destinations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "alert_incidents", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "alert_rule_claims", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "alert_rule_states", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "alert_rules", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "anomaly_detector_settings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "anomaly_detector_states", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "anomaly_incidents", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "api_keys", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "cancellation_reviews", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "chat_identities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "chat_workspaces", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "cloudflare_analytics_state", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "cloudflare_hyperdrive_configs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "cloudflare_logpush_connectors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "gcp_connectors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "cli_device_authorizations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "mcp_oauth_authorizations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "mcp_oauth_clients", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "mcp_oauth_refresh_tokens", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "dashboard_shares", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "dashboard_versions", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "dashboards", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "digest_subscriptions", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "actors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_fingerprint_candidates", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_incidents", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_issue_events", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_issue_pull_requests", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_issue_states", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_issue_verifications", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_issues", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_notification_deliveries", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_notification_policies", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "error_tick_states", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "issue_escalation_policies", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "issue_escalations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "investigations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "oauth_auth_states", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "oauth_connections", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_onboarding_state", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_ingest_attribute_mappings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_recommendation_issues", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_ingest_keys", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_ingest_sampling_policies", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_clickhouse_settings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_clickhouse_schema_apply_runs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "planetscale_connections", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "planetscale_databases", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "planetscale_events", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "planetscale_issue_receipts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "planetscale_poll_state", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "railway_connections", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "railway_environments", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "scrape_target_checks", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "scrape_targets", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "org_support_channels", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "pr_review_edits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "pr_review_finding_embeddings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "pr_review_findings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "pr_review_replies", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "pr_review_settings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "pr_reviews", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "vcs_commits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "vcs_installations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "vcs_repositories", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "vcs_repository_branches", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "impact", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "summary", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reason", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "details", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "related_to", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "agent_type", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "agent_name", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "agent_model", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "agent_version", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "agent_feedback" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "ai_triage_settings" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "ai_triage_settings" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "20", + "generated": null, + "identity": null, + "name": "max_runs_per_day", + "entityType": "columns", + "schema": "public", + "table": "ai_triage_settings" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "90", + "generated": null, + "identity": null, + "name": "max_passes_per_day", + "entityType": "columns", + "schema": "public", + "table": "ai_triage_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "ai_triage_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "ai_triage_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "incident_id", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rule_id", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "destination_id", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "delivery_key", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "event_type", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attempt_number", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scheduled_at", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claimed_at", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claim_expires_at", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claimed_by", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attempted_at", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_message", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_reference", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "response_code", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error_message", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "payload_json", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "config_json", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_iv", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_tag", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_tested_at", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_test_error", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "consecutive_failures", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_failure_at", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "disabled_at", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "disabled_reason", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "alert_destinations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rule_id", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "incident_key", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rule_name", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "group_key", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "signal_type", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "comparator", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "threshold", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "threshold_upper", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "first_triggered_at", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_triggered_at", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolved_at", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_observed_value", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_sample_count", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_evaluated_at", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dedupe_key", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_delivered_event_type", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_notified_at", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hold_reason", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "held_since", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error_issue_id", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "alert_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rule_id", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_claims" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_claims" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_scheduled_at", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_claims" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rule_id", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'__total__'", + "generated": null, + "identity": null, + "name": "group_key", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "consecutive_breaches", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "consecutive_healthy", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_status", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_value", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_sample_count", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_evaluated_at", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "alert_rule_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "notes", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "notification_template_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "service_names_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "exclude_service_names_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environments_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tags_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "signal_type", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "comparator", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "threshold", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "threshold_upper", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "window_minutes", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "minimum_sample_count", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "2", + "generated": null, + "identity": null, + "name": "consecutive_breaches_required", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "2", + "generated": null, + "identity": null, + "name": "consecutive_healthy_required", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "30", + "generated": null, + "identity": null, + "name": "renotify_interval_minutes", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apdex_threshold_ms", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "query_builder_draft_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "raw_query_sql", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "group_by", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "destination_ids_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "query_spec_json", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reducer", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "sample_count_strategy", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "no_data_behavior", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_scheduled_at", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "alert_rules" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'normal'", + "generated": null, + "identity": null, + "name": "sensitivity", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "muted_signals_json", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_tick_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "detector_key", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "signal_type", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "service_name", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "deployment_env", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fingerprint_hash", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "consecutive_breaches", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "consecutive_healthy", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_status", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_value", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "baseline_median", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_sample_count", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_evaluated_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "open_incident_id", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_resolved_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_incident_id", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "detector_key", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "signal_type", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "service_name", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "deployment_env", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fingerprint_hash", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error_issue_id", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "opened_value", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "baseline_median", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "baseline_sigma", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "threshold_value", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_observed_value", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "last_sample_count", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "first_triggered_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_triggered_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolved_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolve_reason", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'none'", + "generated": null, + "identity": null, + "name": "triage_status", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dedupe_key", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "fingerprints_json", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "reopen_count", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_reopened_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "key_hash", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "key_prefix", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "revoked", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_used_at", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata_json", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scopes", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'standard'", + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_email", + "entityType": "columns", + "schema": "public", + "table": "api_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "plan_id", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "bigint", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "subscription_started_at", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "bigint", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "canceled_at", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "snapshot_json", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rule_reason", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "posted_at", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "chat_identities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "chat_identities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector", + "entityType": "columns", + "schema": "public", + "table": "chat_identities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_user_id", + "entityType": "columns", + "schema": "public", + "table": "chat_identities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "chat_identities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "display_name", + "entityType": "columns", + "schema": "public", + "table": "chat_identities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "chat_identities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_workspace_id", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "settings", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credentials_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credentials_iv", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credentials_tag", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "chat_workspaces" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "account_id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dataset", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "zone_id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "zone_name", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "watermark_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "backfill_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "settings_json", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "settings_fetched_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "quantiles_available", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "discovered_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "live_scripts_json", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_success_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "lease_until", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "account_id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "config_id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin_host", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin_port", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin_scheme", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin_database", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin_user", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "deleted_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "zone_name", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "service_name", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'http_requests'", + "generated": null, + "identity": null, + "name": "dataset", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_iv", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_tag", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_hash", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_received_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_rotated_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "project_id", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_iv", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_tag", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "secret_hash", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_received_at", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_code_hash", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_code_hash", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_name", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_org_id", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_user_id", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_roles", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_user_email", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "api_key_id", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_iv", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_tag", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_at", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "denied_at", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "request_id_hash", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_name", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "redirect_uri", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resource", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scopes", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "code_challenge", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "authorization_code_hash", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_org_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_user_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_roles", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_user_email", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "approved_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "denied_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "used_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_clients" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_name", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_clients" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "redirect_uris", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_clients" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_uri", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_clients" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_clients" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_hash", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "family_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resource", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scopes", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "roles", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_email", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "access_key_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "replaced_by_id", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "family_expires_at", + "entityType": "columns", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "live_activity_start_token", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_name", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "disabled_at", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "disabled_reason", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_pushed_at", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_seen_at", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dashboard_id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "widget_id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "mode", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_hash", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_iv", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_tag", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "dashboard_shares" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dashboard_id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "version_number", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "snapshot_json", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "change_kind", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "change_summary", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_version_id", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "dashboard_versions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "payload_json", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "version", + "entityType": "columns", + "schema": "public", + "table": "dashboards" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "opted_out_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "1", + "generated": null, + "identity": null, + "name": "day_of_week", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'UTC'", + "generated": null, + "identity": null, + "name": "timezone", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "namespaces_json", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "environments_json", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_sent_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_attempted_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "web_analytics_enabled", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "web_analytics_opted_out_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "web_analytics_last_sent_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "web_analytics_last_attempted_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "agent_name", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "capabilities_json", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_active_at", + "entityType": "columns", + "schema": "public", + "table": "actors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fingerprint_hash", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "service_name", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "exception_type", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "exception_message", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "error_label", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "top_frame", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "service_versions_json", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "occurrence_count", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "first_seen_at", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_seen_at", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "issue_id", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reason", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "first_triggered_at", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_triggered_at", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolved_at", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "occurrence_count", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_incidents" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "issue_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "actor_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "from_state", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "to_state", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "payload_json", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "issue_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_repo_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repo_full_name", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "number", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_login", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'open'", + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "merged_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "merge_commit_sha", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "link_source", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "linked_by_actor_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "issue_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_observed_occurrence_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_evaluated_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "open_incident_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "issue_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "pull_request_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'waiting'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "merged_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "verify_after", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "baseline_versions_json", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "baseline_occurrence_count", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "baseline_rate_per_hour", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "investigation_id", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "verdict", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "verdict_note", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "post_merge_occurrence_count", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "attempt", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'error'", + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_ref_json", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fingerprint_hash", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "1", + "generated": null, + "identity": null, + "name": "fingerprint_version", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "service_name", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "exception_type", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "exception_message", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "error_label", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "top_frame", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'triage'", + "generated": null, + "identity": null, + "name": "workflow_state", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "3", + "generated": null, + "identity": null, + "name": "priority", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity_source", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "assigned_actor_id", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "lease_holder_actor_id", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "lease_expires_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claimed_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "notes", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "first_seen_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_seen_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "occurrence_count", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolved_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolved_by_actor_id", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_resolved_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_regressed_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "regression_count", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "seen_versions_json", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "resolved_versions_json", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "snooze_until", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "archived_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "destination_id", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "delivery_key", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "payload_json", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "attempt_count", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scheduled_at", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claimed_at", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claim_expires_at", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claimed_by", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attempted_at", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error_message", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "destination_ids_json", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notify_on_first_seen", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notify_on_regression", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "notify_on_resolve", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "notify_on_transition_in_review", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "notify_on_transition_done", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "notify_on_claim", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "1", + "generated": null, + "identity": null, + "name": "min_occurrence_count", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'warning'", + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "error_notification_policies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "error_tick_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "processed_through", + "entityType": "columns", + "schema": "public", + "table": "error_tick_states" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "bootstrap_completed", + "entityType": "columns", + "schema": "public", + "table": "error_tick_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claim_token", + "entityType": "columns", + "schema": "public", + "table": "error_tick_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "claim_expires_at", + "entityType": "columns", + "schema": "public", + "table": "error_tick_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "error_tick_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "issue_escalation_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "issue_escalation_policies" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "rules_json", + "entityType": "columns", + "schema": "public", + "table": "issue_escalation_policies" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "issue_escalation_policies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "issue_escalation_policies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "issue_id", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reason", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "run_id", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "investigation_id", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "payload_json", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "delivery_results_json", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'queued'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "attempts", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dedupe_key", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "processed_at", + "entityType": "columns", + "schema": "public", + "table": "issue_escalations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'investigating'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'user'", + "generated": null, + "identity": null, + "name": "seeded_by", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "subject_json", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "snapshot_json", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "incident_kind", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "incident_id", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "issue_id", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "report_json", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "progress_json", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "confidence", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "input_tokens", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output_tokens", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "autonomous_turns", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "diagnosed_at", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "investigations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "incident_id", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_id", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_reason", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "initiated_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "redirect_uri", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "return_to", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "code_verifier", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_user_id", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_user_email", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_account_name", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "granted_accounts_json", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connected_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "access_token_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "access_token_iv", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "access_token_tag", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refresh_token_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refresh_token_iv", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refresh_token_tag", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "oauth_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "demo_data_requested", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "onboarding_completed_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checklist_dismissed_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "first_data_received_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "welcome_email_sent_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connect_nudge_email_sent_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "stalled_email_sent_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activation_email_sent_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reward_claimed_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reward_reserved_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_onboarding_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_context", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_key", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "target_key", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "operation", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "number", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recommendation_key", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_key", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "canonical_key", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'open'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "usage_count", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "opened_at", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolved_at", + "entityType": "columns", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "public_key", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "public_key_hash", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "private_key_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "private_key_iv", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "private_key_tag", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "private_key_hash", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "public_rotated_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "private_rotated_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_sampling_policies" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "1", + "generated": null, + "identity": null, + "name": "trace_sample_ratio", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_sampling_policies" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "always_keep_error_spans", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_sampling_policies" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "always_keep_slow_spans_ms", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_sampling_policies" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_sampling_policies" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_ingest_sampling_policies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ch_url", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ch_user", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ch_password_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ch_password_iv", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ch_password_tag", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ch_database", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "sync_status", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_sync_at", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_sync_error", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "schema_version", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "workflow_instance_id", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "phase", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "current_migration", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "steps_total", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "steps_done", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_versions", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "skipped", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error_message", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ps_organization", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connected_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scrape_target_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "webhook_secret_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "webhook_secret_iv", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "webhook_secret_tag", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "detected_permissions_json", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_inventory_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_inventory_error", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "database_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'mysql'", + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "region", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "plan", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "branches_json", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "deleted_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_databases" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "database_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "database_name", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "branch_name", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "category", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "event_type", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "external_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "actor_login", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "payload_json", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "occurred_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_issue_receipts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "event_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_issue_receipts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "processed_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_issue_receipts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dataset", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "database_id", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "watermark_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_success_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "lease_until", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_iv", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_tag", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "workspace_names", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connected_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "auth_failed_at", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "discovered_at", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "lease_until", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_success_at", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error_at", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "railway_connections" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connection_id", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "project_id", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "project_name", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_name", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "services_json", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "watermark_at", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_success_at", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_error_at", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "railway_environments" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": { + "type": "byDefault", + "name": "scrape_target_checks_id_seq", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": 1, + "cycle": false + }, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "target_id", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "sub_target_key", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checked_at", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "duration_ms", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "samples_scraped", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "samples_post_relabel", + "entityType": "columns", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "service_name", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'prometheus'", + "generated": null, + "identity": null, + "name": "target_type", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "discovery_config_json", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "15", + "generated": null, + "identity": null, + "name": "scrape_interval_seconds", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "labels_json", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'none'", + "generated": null, + "identity": null, + "name": "auth_type", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "managed_by", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "auth_credentials_ciphertext", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "auth_credentials_iv", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "auth_credentials_tag", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_scrape_at", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_scrape_error", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "scrape_targets" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slack_channel_id", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slack_channel_name", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reservation_id", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reserved_at", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "org_support_channels" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reply_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "seq", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "path", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "old_text", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "new_text", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "pr_review_edits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finding_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_finding_embeddings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_finding_embeddings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_finding_embeddings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model", + "entityType": "columns", + "schema": "public", + "table": "pr_review_finding_embeddings" + }, + { + "type": "real", + "typeSchema": null, + "notNull": true, + "dimensions": 1, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "pr_review_finding_embeddings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "pr_review_finding_embeddings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "number", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "review_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "handle", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "path", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "line", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "category", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "severity", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'open'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "comment_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "resolved_sha", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "reactions_up", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "reactions_down", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "pr_review_findings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "number", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "comment_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "surface", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_root_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_login", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "command", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "head_sha", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'queued'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "session_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reply_url", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "commit_sha", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "pr_review_replies" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "pr_review_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model", + "entityType": "columns", + "schema": "public", + "table": "pr_review_settings" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "defaults", + "entityType": "columns", + "schema": "public", + "table": "pr_review_settings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "pr_review_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_by", + "entityType": "columns", + "schema": "public", + "table": "pr_review_settings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_id", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "number", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "head_sha", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "base_sha", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_login", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'queued'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "skip_reason", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "session_id", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "comment_attempt", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "report_json", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "score", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "check_run_url", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "comment_url", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "review_url", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "publish_error", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "input_tokens", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output_tokens", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "merged_at", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "telemetry_json", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "merge_commit_sha", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_merge_status", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_merge_after", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_merge_json", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "pr_reviews" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "sha", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "message", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_name", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_email", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_login", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_avatar_url", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "authored_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "committed_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "html_url", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_commits" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_installation_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "account_login", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "account_type", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_account_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "account_avatar_url", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'all'", + "generated": null, + "identity": null, + "name": "repository_selection", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suspended_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "installed_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "installation_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_repo_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "owner", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "full_name", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'main'", + "generated": null, + "identity": null, + "name": "default_branch", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tracked_branch", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "html_url", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "is_private", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "is_archived", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "sync_status", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_synced_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_sync_error", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "pr_review_enabled", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "pr_review_config", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_repositories" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "org_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_id", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "is_default", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "head_sha", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "agent_feedback_org_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "agent_feedback" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "kind", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "agent_feedback_kind_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "agent_feedback" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_delivery_events_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "incident_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_delivery_events_org_incident_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "scheduled_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_delivery_events_due_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "claim_expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "scheduled_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_delivery_events_claim_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "delivery_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "attempt_number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_delivery_events_delivery_attempt_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_delivery_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_destinations_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_destinations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "enabled", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_destinations_org_enabled_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_destinations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_destinations_org_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_destinations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_incidents_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_incidents_org_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "rule_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_incidents_org_rule_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "error_issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_incidents_org_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "incident_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_incidents_incident_key_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "rule_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "group_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"status\" = 'open'", + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_incidents_open_group_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_rule_claims_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_rule_claims" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_rule_states_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_rule_states" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_rules_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_rules" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "enabled", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_rules_org_enabled_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_rules" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "alert_rules_org_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "alert_rules" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "open_incident_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"open_incident_id\" is not null", + "with": "", + "method": "btree", + "concurrently": false, + "name": "anomaly_detector_states_open_incident_idx", + "entityType": "indexes", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "last_evaluated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "anomaly_detector_states_evaluated_idx", + "entityType": "indexes", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "last_triggered_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "anomaly_incidents_org_status_triggered_idx", + "entityType": "indexes", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "last_triggered_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "anomaly_incidents_org_triggered_idx", + "entityType": "indexes", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "detector_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "anomaly_incidents_org_detector_idx", + "entityType": "indexes", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "error_issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "anomaly_incidents_org_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "detector_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"status\" = 'open'", + "with": "", + "method": "btree", + "concurrently": false, + "name": "anomaly_incidents_open_detector_idx", + "entityType": "indexes", + "schema": "public", + "table": "anomaly_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "key_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "api_keys_key_hash_unique", + "entityType": "indexes", + "schema": "public", + "table": "api_keys" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "api_keys_org_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "api_keys" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "plan_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "subscription_started_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cancellation_reviews_subscription_idx", + "entityType": "indexes", + "schema": "public", + "table": "cancellation_reviews" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "connector", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "external_user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "chat_identities_org_connector_external_idx", + "entityType": "indexes", + "schema": "public", + "table": "chat_identities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "connector", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "chat_identities_org_connector_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "chat_identities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "chat_identities_org_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "chat_identities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "external_workspace_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "chat_workspaces_connector_external_idx", + "entityType": "indexes", + "schema": "public", + "table": "chat_workspaces" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "chat_workspaces_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "chat_workspaces" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "account_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "dataset", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "zone_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cf_analytics_state_org_account_dataset_zone_idx", + "entityType": "indexes", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cf_analytics_state_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "cloudflare_analytics_state" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "config_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cloudflare_hyperdrive_configs_org_config_idx", + "entityType": "indexes", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cloudflare_hyperdrive_configs_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "cloudflare_hyperdrive_configs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cloudflare_logpush_connectors_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "enabled", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cloudflare_logpush_connectors_org_enabled_idx", + "entityType": "indexes", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "secret_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cloudflare_logpush_connectors_secret_hash_unique", + "entityType": "indexes", + "schema": "public", + "table": "cloudflare_logpush_connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "gcp_connectors_org_project_idx", + "entityType": "indexes", + "schema": "public", + "table": "gcp_connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "secret_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "gcp_connectors_secret_hash_unique", + "entityType": "indexes", + "schema": "public", + "table": "gcp_connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_code_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cli_device_authorizations_user_code_unique", + "entityType": "indexes", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "cli_device_authorizations_expires_idx", + "entityType": "indexes", + "schema": "public", + "table": "cli_device_authorizations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "authorization_code_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mcp_oauth_authorizations_code_unique", + "entityType": "indexes", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mcp_oauth_authorizations_expires_idx", + "entityType": "indexes", + "schema": "public", + "table": "mcp_oauth_authorizations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "token_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mcp_oauth_refresh_tokens_hash_unique", + "entityType": "indexes", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "family_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mcp_oauth_refresh_tokens_family_idx", + "entityType": "indexes", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mcp_oauth_refresh_tokens_expires_idx", + "entityType": "indexes", + "schema": "public", + "table": "mcp_oauth_refresh_tokens" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "platform", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mobile_devices_org_platform_token_unique", + "entityType": "indexes", + "schema": "public", + "table": "mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mobile_devices_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "mobile_devices_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "token_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboard_shares_token_hash_unq", + "entityType": "indexes", + "schema": "public", + "table": "dashboard_shares" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "dashboard_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "coalesce(widget_id, '')", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "revoked_at is null", + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboard_shares_live_unq", + "entityType": "indexes", + "schema": "public", + "table": "dashboard_shares" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "dashboard_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboard_shares_org_dashboard_idx", + "entityType": "indexes", + "schema": "public", + "table": "dashboard_shares" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboard_shares_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "dashboard_shares" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "dashboard_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "version_number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboard_versions_org_dashboard_idx", + "entityType": "indexes", + "schema": "public", + "table": "dashboard_versions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "dashboard_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "version_number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboard_versions_org_dashboard_version_unq", + "entityType": "indexes", + "schema": "public", + "table": "dashboard_versions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboards_org_updated_idx", + "entityType": "indexes", + "schema": "public", + "table": "dashboards" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "dashboards_org_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "dashboards" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "digest_subscriptions_org_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "enabled", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "digest_subscriptions_org_enabled_idx", + "entityType": "indexes", + "schema": "public", + "table": "digest_subscriptions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "actors_org_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "actors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "agent_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "actors_org_agent_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "actors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "actors_org_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "actors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "last_seen_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_fingerprint_candidates_last_seen_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_incidents_org_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "last_triggered_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_incidents_org_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_incidents" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_events_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "actor_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_events_actor_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_events_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "repo_full_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_pull_requests_issue_pr_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "repo_full_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_pull_requests_repo_number_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_pull_requests_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_pull_requests" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "verify_after", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_verifications_due_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_verifications_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"status\" in ('waiting', 'running')", + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issue_verifications_open_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issue_verifications" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "fingerprint_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_org_fp_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "workflow_state", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_org_workflow_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "severity", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_org_severity_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "last_seen_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" is null", + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_org_live_seen_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "fingerprint_version", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_org_fp_version_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "assigned_actor_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_org_assignee_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "lease_expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_lease_expiry_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "archived_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" is not null", + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_issues_org_archived_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "scheduled_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "claim_expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_notification_deliveries_due_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_notification_deliveries_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "delivery_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "destination_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_notification_deliveries_key_destination_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_notification_deliveries" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "claim_expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "error_tick_states_claim_idx", + "entityType": "indexes", + "schema": "public", + "table": "error_tick_states" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "dedupe_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "issue_escalations_dedupe_idx", + "entityType": "indexes", + "schema": "public", + "table": "issue_escalations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "issue_escalations_due_idx", + "entityType": "indexes", + "schema": "public", + "table": "issue_escalations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "issue_escalations_org_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "issue_escalations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "incident_kind", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "incident_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"incident_id\" is not null", + "with": "", + "method": "btree", + "concurrently": false, + "name": "investigations_incident_idx", + "entityType": "indexes", + "schema": "public", + "table": "investigations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "investigations_org_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "investigations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "issue_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "investigations_org_issue_idx", + "entityType": "indexes", + "schema": "public", + "table": "investigations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "investigations_org_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "investigations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "device_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "incident_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "live_activities_device_incident_unique", + "entityType": "indexes", + "schema": "public", + "table": "live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "incident_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "live_activities_incident_idx", + "entityType": "indexes", + "schema": "public", + "table": "live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "oauth_auth_states_expires_idx", + "entityType": "indexes", + "schema": "public", + "table": "oauth_auth_states" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "oauth_connections_org_provider_idx", + "entityType": "indexes", + "schema": "public", + "table": "oauth_connections" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "oauth_connections_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "oauth_connections" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "org_ingest_attribute_mappings_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "org_ingest_attribute_mappings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "org_recommendation_issues_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "recommendation_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "org_recommendation_issues_org_key_idx", + "entityType": "indexes", + "schema": "public", + "table": "org_recommendation_issues" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "org_ingest_keys_public_key_unique", + "entityType": "indexes", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "public_key_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "org_ingest_keys_public_key_hash_unique", + "entityType": "indexes", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "private_key_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "org_ingest_keys_private_key_hash_unique", + "entityType": "indexes", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_connections_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_connections" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "database_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_databases_org_db_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_databases" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_databases_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_databases" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "database_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "event_type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "external_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "occurred_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_events_dedupe_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "database_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "occurred_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_events_org_db_time_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "occurred_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_events_org_time_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "processed_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_issue_receipts_processed_at_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_issue_receipts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "dataset", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "database_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_poll_state_org_dataset_db_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "planetscale_poll_state_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "planetscale_poll_state" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "railway_connections_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "railway_connections" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "railway_environments_org_env_idx", + "entityType": "indexes", + "schema": "public", + "table": "railway_environments" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connection_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "railway_environments_connection_idx", + "entityType": "indexes", + "schema": "public", + "table": "railway_environments" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "target_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "checked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "scrape_target_checks_target_checked_idx", + "entityType": "indexes", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "scrape_targets_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "scrape_targets" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "enabled", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "scrape_targets_org_enabled_idx", + "entityType": "indexes", + "schema": "public", + "table": "scrape_targets" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "reply_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "seq", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_review_edits_reply_seq_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_review_edits" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "model", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_review_finding_embeddings_org_model_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_review_finding_embeddings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "handle", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_review_findings_pr_handle_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_review_findings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_review_findings_org_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_review_findings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "comment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_review_replies_repo_comment_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_review_replies" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_review_replies_org_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_review_replies" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "head_sha", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_reviews_repo_number_head_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_reviews" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "number", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_reviews_repo_number_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_reviews" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_reviews_org_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_reviews" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "post_merge_after", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"post_merge_status\" = 'waiting'", + "with": "", + "method": "btree", + "concurrently": false, + "name": "pr_reviews_post_merge_due_idx", + "entityType": "indexes", + "schema": "public", + "table": "pr_reviews" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "sha", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_commits_repo_sha_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_commits" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "sha", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_commits_org_sha_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_commits" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "external_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_installations_provider_external_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_installations_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "external_repo_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_repositories_org_repo_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_repositories" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_repositories_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_repositories" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_repositories_installation_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_repositories" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_repository_branches_repo_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "org_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "vcs_repository_branches_org_idx", + "entityType": "indexes", + "schema": "public", + "table": "vcs_repository_branches" + }, + { + "nameExplicit": true, + "columns": [ + "org_id", + "dashboard_id" + ], + "schemaTo": "public", + "tableTo": "dashboards", + "columnsTo": [ + "org_id", + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "dashboard_shares_dashboard_fk", + "entityType": "fks", + "schema": "public", + "table": "dashboard_shares" + }, + { + "nameExplicit": false, + "columns": [ + "target_id" + ], + "schemaTo": "public", + "tableTo": "scrape_targets", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "scrape_target_checks_target_id_scrape_targets_id_fk", + "entityType": "fks", + "schema": "public", + "table": "scrape_target_checks" + }, + { + "columns": [ + "org_id", + "rule_id", + "group_key" + ], + "nameExplicit": false, + "name": "alert_rule_states_org_id_rule_id_group_key_pk", + "entityType": "pks", + "schema": "public", + "table": "alert_rule_states" + }, + { + "columns": [ + "org_id", + "detector_key" + ], + "nameExplicit": false, + "name": "anomaly_detector_states_org_id_detector_key_pk", + "entityType": "pks", + "schema": "public", + "table": "anomaly_detector_states" + }, + { + "columns": [ + "org_id", + "id" + ], + "nameExplicit": false, + "name": "dashboard_shares_org_id_id_pk", + "entityType": "pks", + "schema": "public", + "table": "dashboard_shares" + }, + { + "columns": [ + "org_id", + "id" + ], + "nameExplicit": false, + "name": "dashboard_versions_org_id_id_pk", + "entityType": "pks", + "schema": "public", + "table": "dashboard_versions" + }, + { + "columns": [ + "org_id", + "id" + ], + "nameExplicit": false, + "name": "dashboards_org_id_id_pk", + "entityType": "pks", + "schema": "public", + "table": "dashboards" + }, + { + "columns": [ + "org_id", + "fingerprint_hash" + ], + "nameExplicit": false, + "name": "error_fingerprint_candidates_org_id_fingerprint_hash_pk", + "entityType": "pks", + "schema": "public", + "table": "error_fingerprint_candidates" + }, + { + "columns": [ + "org_id", + "issue_id" + ], + "nameExplicit": false, + "name": "error_issue_states_org_id_issue_id_pk", + "entityType": "pks", + "schema": "public", + "table": "error_issue_states" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "org_ingest_keys_org_id_pk", + "entityType": "pks", + "schema": "public", + "table": "org_ingest_keys" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "org_clickhouse_settings_org_id_pk", + "entityType": "pks", + "schema": "public", + "table": "org_clickhouse_settings" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "org_clickhouse_schema_apply_runs_org_id_pk", + "entityType": "pks", + "schema": "public", + "table": "org_clickhouse_schema_apply_runs" + }, + { + "columns": [ + "org_id", + "event_id" + ], + "nameExplicit": false, + "name": "planetscale_issue_receipts_org_id_event_id_pk", + "entityType": "pks", + "schema": "public", + "table": "planetscale_issue_receipts" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "agent_feedback_pkey", + "schema": "public", + "table": "agent_feedback", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "ai_triage_settings_pkey", + "schema": "public", + "table": "ai_triage_settings", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "alert_delivery_events_pkey", + "schema": "public", + "table": "alert_delivery_events", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "alert_destinations_pkey", + "schema": "public", + "table": "alert_destinations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "alert_incidents_pkey", + "schema": "public", + "table": "alert_incidents", + "entityType": "pks" + }, + { + "columns": [ + "rule_id" + ], + "nameExplicit": false, + "name": "alert_rule_claims_pkey", + "schema": "public", + "table": "alert_rule_claims", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "alert_rules_pkey", + "schema": "public", + "table": "alert_rules", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "anomaly_detector_settings_pkey", + "schema": "public", + "table": "anomaly_detector_settings", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "anomaly_incidents_pkey", + "schema": "public", + "table": "anomaly_incidents", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "api_keys_pkey", + "schema": "public", + "table": "api_keys", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "cancellation_reviews_pkey", + "schema": "public", + "table": "cancellation_reviews", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "chat_identities_pkey", + "schema": "public", + "table": "chat_identities", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "chat_workspaces_pkey", + "schema": "public", + "table": "chat_workspaces", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "cloudflare_analytics_state_pkey", + "schema": "public", + "table": "cloudflare_analytics_state", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "cloudflare_hyperdrive_configs_pkey", + "schema": "public", + "table": "cloudflare_hyperdrive_configs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "cloudflare_logpush_connectors_pkey", + "schema": "public", + "table": "cloudflare_logpush_connectors", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "gcp_connectors_pkey", + "schema": "public", + "table": "gcp_connectors", + "entityType": "pks" + }, + { + "columns": [ + "device_code_hash" + ], + "nameExplicit": false, + "name": "cli_device_authorizations_pkey", + "schema": "public", + "table": "cli_device_authorizations", + "entityType": "pks" + }, + { + "columns": [ + "request_id_hash" + ], + "nameExplicit": false, + "name": "mcp_oauth_authorizations_pkey", + "schema": "public", + "table": "mcp_oauth_authorizations", + "entityType": "pks" + }, + { + "columns": [ + "client_id" + ], + "nameExplicit": false, + "name": "mcp_oauth_clients_pkey", + "schema": "public", + "table": "mcp_oauth_clients", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "mcp_oauth_refresh_tokens_pkey", + "schema": "public", + "table": "mcp_oauth_refresh_tokens", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "mobile_devices_pkey", + "schema": "public", + "table": "mobile_devices", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "digest_subscriptions_pkey", + "schema": "public", + "table": "digest_subscriptions", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "actors_pkey", + "schema": "public", + "table": "actors", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "error_incidents_pkey", + "schema": "public", + "table": "error_incidents", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "error_issue_events_pkey", + "schema": "public", + "table": "error_issue_events", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "error_issue_pull_requests_pkey", + "schema": "public", + "table": "error_issue_pull_requests", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "error_issue_verifications_pkey", + "schema": "public", + "table": "error_issue_verifications", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "error_issues_pkey", + "schema": "public", + "table": "error_issues", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "error_notification_deliveries_pkey", + "schema": "public", + "table": "error_notification_deliveries", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "error_notification_policies_pkey", + "schema": "public", + "table": "error_notification_policies", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "error_tick_states_pkey", + "schema": "public", + "table": "error_tick_states", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "issue_escalation_policies_pkey", + "schema": "public", + "table": "issue_escalation_policies", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "issue_escalations_pkey", + "schema": "public", + "table": "issue_escalations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "investigations_pkey", + "schema": "public", + "table": "investigations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "live_activities_pkey", + "schema": "public", + "table": "live_activities", + "entityType": "pks" + }, + { + "columns": [ + "state" + ], + "nameExplicit": false, + "name": "oauth_auth_states_pkey", + "schema": "public", + "table": "oauth_auth_states", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "oauth_connections_pkey", + "schema": "public", + "table": "oauth_connections", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "org_onboarding_state_pkey", + "schema": "public", + "table": "org_onboarding_state", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "org_ingest_attribute_mappings_pkey", + "schema": "public", + "table": "org_ingest_attribute_mappings", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "org_recommendation_issues_pkey", + "schema": "public", + "table": "org_recommendation_issues", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "org_ingest_sampling_policies_pkey", + "schema": "public", + "table": "org_ingest_sampling_policies", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "planetscale_connections_pkey", + "schema": "public", + "table": "planetscale_connections", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "planetscale_databases_pkey", + "schema": "public", + "table": "planetscale_databases", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "planetscale_events_pkey", + "schema": "public", + "table": "planetscale_events", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "planetscale_poll_state_pkey", + "schema": "public", + "table": "planetscale_poll_state", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "railway_connections_pkey", + "schema": "public", + "table": "railway_connections", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "railway_environments_pkey", + "schema": "public", + "table": "railway_environments", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "scrape_target_checks_pkey", + "schema": "public", + "table": "scrape_target_checks", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "scrape_targets_pkey", + "schema": "public", + "table": "scrape_targets", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "org_support_channels_pkey", + "schema": "public", + "table": "org_support_channels", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "pr_review_edits_pkey", + "schema": "public", + "table": "pr_review_edits", + "entityType": "pks" + }, + { + "columns": [ + "finding_id" + ], + "nameExplicit": false, + "name": "pr_review_finding_embeddings_pkey", + "schema": "public", + "table": "pr_review_finding_embeddings", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "pr_review_findings_pkey", + "schema": "public", + "table": "pr_review_findings", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "pr_review_replies_pkey", + "schema": "public", + "table": "pr_review_replies", + "entityType": "pks" + }, + { + "columns": [ + "org_id" + ], + "nameExplicit": false, + "name": "pr_review_settings_pkey", + "schema": "public", + "table": "pr_review_settings", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "pr_reviews_pkey", + "schema": "public", + "table": "pr_reviews", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "vcs_commits_pkey", + "schema": "public", + "table": "vcs_commits", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "vcs_installations_pkey", + "schema": "public", + "table": "vcs_installations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "vcs_repositories_pkey", + "schema": "public", + "table": "vcs_repositories", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "vcs_repository_branches_pkey", + "schema": "public", + "table": "vcs_repository_branches", + "entityType": "pks" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/db/src/schema/gcp-connectors.ts b/packages/db/src/schema/gcp-connectors.ts new file mode 100644 index 0000000000..adc91d8dd8 --- /dev/null +++ b/packages/db/src/schema/gcp-connectors.ts @@ -0,0 +1,28 @@ +import type { OrgId } from "@maple/domain" +import { pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" + +// One row per connected Google Cloud project. The ingest gateway's `/v1/logpush/gcp/...` receiver +// authenticates a push by `id` + `secret_hash` and writes `last_received_at` / `last_error`. +export const gcpConnectors = pgTable( + "gcp_connectors", + { + id: text("id").notNull().primaryKey(), + orgId: text("org_id").$type().notNull(), + projectId: text("project_id").notNull(), + secretCiphertext: text("secret_ciphertext").notNull(), + secretIv: text("secret_iv").notNull(), + secretTag: text("secret_tag").notNull(), + secretHash: text("secret_hash").notNull(), + lastReceivedAt: timestamp("last_received_at", { withTimezone: true, mode: "date" }), + lastError: text("last_error"), + createdBy: text("created_by").notNull(), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }).notNull(), + }, + (table) => [ + uniqueIndex("gcp_connectors_org_project_idx").on(table.orgId, table.projectId), + uniqueIndex("gcp_connectors_secret_hash_unique").on(table.secretHash), + ], +) + +export type GcpConnectorRow = typeof gcpConnectors.$inferSelect diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 34e438e60c..5733038591 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -9,6 +9,7 @@ export * from "./chat-workspaces" export * from "./cloudflare-analytics-state" export * from "./cloudflare-hyperdrive-configs" export * from "./cloudflare-logpush-connectors" +export * from "./gcp-connectors" export * from "./cli-device-authorizations" export * from "./mcp-oauth" export * from "./mobile-devices" diff --git a/packages/domain/package.json b/packages/domain/package.json index 4fb87ea2d7..927cf6cb70 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -11,6 +11,7 @@ "./billing": "./src/billing.ts", "./billing-tax-ids": "./src/billing-tax-ids.ts", "./format": "./src/format.ts", + "./gcp": "./src/gcp.ts", "./gen-ai": "./src/gen-ai.ts", "./glob": "./src/glob.ts", "./http": "./src/http/index.ts", diff --git a/packages/domain/src/gcp.test.ts b/packages/domain/src/gcp.test.ts new file mode 100644 index 0000000000..54419e700a --- /dev/null +++ b/packages/domain/src/gcp.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest" +import { Schema } from "effect" +import { gcpConnectorResourceNames } from "./gcp" +import { GcpConnectorId } from "./primitives" + +const connectorId = Schema.decodeUnknownSync(GcpConnectorId) + +describe("gcpConnectorResourceNames", () => { + it("derives a service account id inside Google's limits from the connector id alone", () => { + const names = gcpConnectorResourceNames(connectorId("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8")) + expect(names).toEqual({ + serviceAccountId: "maple-018f2b3c4d5e4f708192a3b4", + topic: "maple-018f2b3c4d5e4f708192a3b4", + subscription: "maple-018f2b3c4d5e4f708192a3b4", + sink: "maple-018f2b3c4d5e4f708192a3b4", + }) + // Google: 6-30 characters, [a-z]([-a-z0-9]*[a-z0-9]). + expect(names.serviceAccountId).toMatch(/^[a-z][-a-z0-9]{4,28}[a-z0-9]$/) + }) + + it("gives two connectors different names", () => { + expect(gcpConnectorResourceNames(connectorId("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8"))).not.toEqual( + gcpConnectorResourceNames(connectorId("118f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8")), + ) + }) +}) diff --git a/packages/domain/src/gcp.ts b/packages/domain/src/gcp.ts new file mode 100644 index 0000000000..8404de0012 --- /dev/null +++ b/packages/domain/src/gcp.ts @@ -0,0 +1,14 @@ +import type { GcpConnectorId } from "./primitives" + +/** + * Names of the resources a connector's setup script creates in the customer's Google Cloud + * project. Derived from the connector id alone, so nothing extra is stored and the names cannot + * be chosen by a caller: the metrics reader of one connector is never the account another + * organization's connector impersonates. + * + * 24 hex characters of the id keep the service account id at Google's 30-character limit. + */ +export const gcpConnectorResourceNames = (connectorId: GcpConnectorId) => { + const name = `maple-${connectorId.replaceAll("-", "").slice(0, 24)}` + return { serviceAccountId: name, topic: name, subscription: name, sink: name } +} diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index d99a76c1a4..3b992896ed 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -47,6 +47,7 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/ErrorIssuePullRequestNotFoundError", "@maple/http/errors/ErrorIssueTransitionError", "@maple/http/errors/ErrorValidationError", + "@maple/http/errors/GcpProjectAlreadyConnectedError", "@maple/http/errors/IncidentTriageUnauthorizedError", "@maple/http/errors/IngestAttributeMappingForbiddenError", "@maple/http/errors/IngestAttributeMappingNotFoundError", diff --git a/packages/domain/src/http/integrations.ts b/packages/domain/src/http/integrations.ts index 2178c2a9b6..59af5eaf45 100644 --- a/packages/domain/src/http/integrations.ts +++ b/packages/domain/src/http/integrations.ts @@ -1,6 +1,6 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" import { Schema } from "effect" -import { ExternalUserId, ScrapeTargetId, UserId } from "../primitives" +import { ExternalUserId, GcpProjectId, ScrapeTargetId, UserId } from "../primitives" import { Authorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" import { PrReviewListItem, PrReviewOrgSettings, PrReviewRepositoryConfig } from "./pr-review" @@ -1043,6 +1043,24 @@ export class IntegrationsNotFoundError extends HttpTaggedError()( + "@maple/http/errors/GcpProjectAlreadyConnectedError", + { + projectId: GcpProjectId, + message: Schema.String, + }, + { + status: 409, + code: "gcp_project_already_connected", + title: "Project already connected", + param: "project_id", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + export type IntegrationHttpError = | IntegrationsForbiddenError | IntegrationsConfigurationError diff --git a/packages/domain/src/http/v2/api.ts b/packages/domain/src/http/v2/api.ts index d3195edd4c..81c0be2ebf 100644 --- a/packages/domain/src/http/v2/api.ts +++ b/packages/domain/src/http/v2/api.ts @@ -10,6 +10,7 @@ import { V2AuditLogApiGroup } from "./audit-log" import { V2DashboardsApiGroup } from "./dashboards" import { V2IngestKeysApiGroup } from "./ingest-keys" import { V2ChatIntegrationsApiGroup } from "./integrations-chat" +import { V2GcpIntegrationsApiGroup } from "./integrations-gcp" import { V2PlanetScaleIntegrationsApiGroup } from "./integrations-planetscale" import { V2ErrorIssuesApiGroup } from "./error-issues" import { V2InvestigationsApiGroup } from "./investigations" @@ -97,6 +98,7 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") .add(V2IngestKeysApiGroup) .add(V2ChatIntegrationsApiGroup) .add(V2PlanetScaleIntegrationsApiGroup) + .add(V2GcpIntegrationsApiGroup) .add(V2ErrorIssuesApiGroup) .add(V2AttributeMappingsApiGroup) .add(V2AuditLogApiGroup) diff --git a/packages/domain/src/http/v2/index.ts b/packages/domain/src/http/v2/index.ts index 1ddbd9d5f9..11f7b80266 100644 --- a/packages/domain/src/http/v2/index.ts +++ b/packages/domain/src/http/v2/index.ts @@ -14,6 +14,7 @@ export * from "./errors" export * from "./error-issues" export * from "./ingest-keys" export * from "./integrations-chat" +export * from "./integrations-gcp" export * from "./integrations-planetscale" export * from "./investigations" export * from "./agent-feedback" diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts new file mode 100644 index 0000000000..374b956166 --- /dev/null +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -0,0 +1,238 @@ +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api" +import { Schema } from "effect" +import { GcpConnectorId, GcpProjectId } from "../../primitives" +import { + GcpProjectAlreadyConnectedError, + IntegrationsNotFoundError, + IntegrationsPersistenceError, +} from "../integrations" +import { AuthorizationV2 } from "./auth" +import { wireExample, Timestamp } from "./envelopes" +import { V2InsufficientPermissions } from "./errors" +import { publicErrors } from "./public-error" +import { PublicId, PublicIdPrefixes } from "./public-id" + +// Google Cloud integration. There is no OAuth: an org registers a connector for one Google Cloud +// project and its owner runs the generated `gcloud` script in Cloud Shell. The script routes the +// project's logs through Pub/Sub to Maple's ingest gateway and, where this deployment has a Google +// identity, creates a read-only metrics account Maple may impersonate. +// +// Public v2 from the start: infrastructure-as-code setups create connectors with a scoped API key, +// which only v2 has (docs/api-v2.md). The log receiver itself is a provider-facing route on the +// ingest gateway, not part of this group. + +/** `gcpc_…` public ID ⇄ internal `GcpConnectorId` (raw UUID). */ +export const GcpConnectorPublicId = PublicId(PublicIdPrefixes.gcpConnector, GcpConnectorId) + +const connectorExample = { + id: "gcpc_YofPTrK9782DWwcnXhpcCw", + object: "gcp_connector", + project_id: "acme-prod", + created_at: "2026-10-01T12:00:00.000Z", + last_log_received_at: "2026-10-08T09:12:00.000Z", + last_log_error: null, +} as const + +export const V2GcpConnector = Schema.Struct({ + id: GcpConnectorPublicId, + object: Schema.Literal("gcp_connector").annotate({ + description: 'The object type — always `"gcp_connector"`.', + }), + project_id: Schema.String.annotate({ + description: "The Google Cloud project ID this connector receives from.", + examples: ["acme-prod"], + }), + created_at: Timestamp.annotate({ description: "When the connector was created." }), + last_log_received_at: Schema.NullOr(Timestamp).annotate({ + description: + "When Maple last accepted a log push from this project, or `null` before the first one. Stays `null` until the setup script has run.", + }), + last_log_error: Schema.NullOr(Schema.String).annotate({ + description: "Why the most recent log push was rejected, or `null`.", + examples: [null], + }), +}).annotate({ + identifier: "GcpConnector", + title: "Google Cloud connector", + description: "One connected Google Cloud project.", + examples: [wireExample(connectorExample)], +}) +export type V2GcpConnector = Schema.Schema.Type + +export const V2GcpIntegration = Schema.Struct({ + object: Schema.Literal("gcp_integration").annotate({ + description: 'The object type — always `"gcp_integration"`.', + }), + metrics_available: Schema.Boolean.annotate({ + description: + "Whether this Maple deployment can collect Cloud Monitoring metrics. When `false` connectors are logs-only and the setup script omits the metrics steps.", + examples: [true], + }), + connectors: Schema.Array(V2GcpConnector).annotate({ + description: "Every connector of the organization, oldest first.", + }), +}).annotate({ + identifier: "GcpIntegration", + title: "Google Cloud integration status", + description: + 'The Google Cloud integration state for the authenticated organization. Note: `connectors` is **not** the standard `{ object: "list", data, has_more, next_cursor }` envelope: an organization connects a handful of projects and they are returned whole next to `metrics_available`.', + examples: [ + wireExample({ object: "gcp_integration", metrics_available: true, connectors: [connectorExample] }), + ], +}) +export type V2GcpIntegration = Schema.Schema.Type + +export const V2GcpCreateConnectorRequest = Schema.Struct({ + project_id: GcpProjectId.annotate({ + description: + "The Google Cloud project ID (not its name or number): 6 to 30 lowercase letters, digits and hyphens, starting with a letter.", + }), +}).annotate({ + identifier: "GcpCreateConnectorRequest", + title: "Google Cloud connector create request", + description: "The project to connect. Connect several projects by creating one connector each.", + examples: [wireExample({ project_id: "acme-prod" })], +}) +export type V2GcpCreateConnectorRequest = Schema.Schema.Type + +export const V2GcpSetupScriptsRequest = Schema.Struct({ + exclude_gke_container_logs: Schema.optionalKey(Schema.Boolean).annotate({ + description: + "Leave GKE container logs out of the log filter. Set it when the project's pods already ship their logs to Maple through an OpenTelemetry collector, so they are not ingested twice. Defaults to `false`.", + examples: [false], + }), +}).annotate({ + identifier: "GcpSetupScriptsRequest", + title: "Google Cloud setup scripts request", + description: "Options for rendering a connector's scripts.", + examples: [wireExample({ exclude_gke_container_logs: true })], +}) +export type V2GcpSetupScriptsRequest = Schema.Schema.Type + +export const V2GcpSetupScripts = Schema.Struct({ + object: Schema.Literal("gcp_connector.setup_scripts").annotate({ + description: 'The object type — always `"gcp_connector.setup_scripts"`.', + }), + setup_script: Schema.String.annotate({ + description: + "A bash script for Cloud Shell, run by a project owner. It embeds the connector's push secret, so treat it as a credential. Safe to re-run; the log filter is an editable variable at the top.", + examples: ["#!/usr/bin/env bash\nset -euo pipefail\n…"], + }), + cleanup_script: Schema.String.annotate({ + description: "A bash script that removes everything the setup script created. Carries no secret.", + examples: ["#!/usr/bin/env bash\nset -euo pipefail\n…"], + }), +}).annotate({ + identifier: "GcpSetupScripts", + title: "Google Cloud setup scripts", + description: "The scripts that connect and disconnect one Google Cloud project.", + examples: [ + wireExample({ + object: "gcp_connector.setup_scripts", + setup_script: "#!/usr/bin/env bash\nset -euo pipefail\n…", + cleanup_script: "#!/usr/bin/env bash\nset -euo pipefail\n…", + }), + ], +}) +export type V2GcpSetupScripts = Schema.Schema.Type + +export const V2GcpConnectorDeleteResponse = Schema.Struct({ + id: GcpConnectorPublicId, + object: Schema.Literal("gcp_connector").annotate({ + description: 'The object type — always `"gcp_connector"`.', + }), + deleted: Schema.Literal(true).annotate({ + description: "Always `true` — the connector no longer exists and Maple rejects its log pushes.", + }), + cleanup_script: Schema.String.annotate({ + description: + "A bash script that removes the resources the setup script created. Maple has no write access to the project, so the project owner runs it.", + examples: ["#!/usr/bin/env bash\nset -euo pipefail\n…"], + }), +}).annotate({ + identifier: "GcpConnectorDeleteResponse", + title: "Google Cloud connector delete response", + description: "Confirmation that a connector was deleted, with the script that cleans up the project.", + examples: [ + wireExample({ + id: connectorExample.id, + object: "gcp_connector", + deleted: true, + cleanup_script: "#!/usr/bin/env bash\nset -euo pipefail\n…", + }), + ], +}) +export type V2GcpConnectorDeleteResponse = Schema.Schema.Type + +const [projectAlreadyConnected, connectorNotFound, integrationPersistence] = publicErrors( + GcpProjectAlreadyConnectedError, + IntegrationsNotFoundError, + IntegrationsPersistenceError, +) + +export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration") + .add( + HttpApiEndpoint.get("status", "/", { + success: V2GcpIntegration, + error: [integrationPersistence], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getGcpIntegration", + summary: "Retrieve Google Cloud integration status", + description: + "Returns the organization's connected Google Cloud projects with the time and outcome of their latest log push, and whether metrics collection is available. Requires the `integrations:read` scope.", + }), + ), + ) + .add( + HttpApiEndpoint.post("createConnector", "/connectors", { + payload: V2GcpCreateConnectorRequest, + success: V2GcpConnector, + error: [V2InsufficientPermissions.schema, projectAlreadyConnected, integrationPersistence], + }).annotateMerge( + OpenApi.annotations({ + identifier: "createGcpConnector", + summary: "Connect a Google Cloud project", + description: + "Registers a connector for one Google Cloud project. Nothing reaches Maple until the project owner runs the script from `setup_scripts`. A project can be connected once per organization. Requires an org-admin role and the `integrations:write` scope.", + }), + ), + ) + .add( + HttpApiEndpoint.post("setupScripts", "/connectors/:id/setup_scripts", { + params: { id: GcpConnectorPublicId }, + payload: V2GcpSetupScriptsRequest, + success: V2GcpSetupScripts, + error: [V2InsufficientPermissions.schema, connectorNotFound, integrationPersistence], + }).annotateMerge( + OpenApi.annotations({ + identifier: "renderGcpConnectorSetupScripts", + summary: "Render a connector's setup and cleanup scripts", + description: + "Returns the `gcloud` scripts for one connector. Rendering changes nothing, but the setup script carries the connector's push secret, so this requires an org-admin role and the `integrations:write` scope.", + }), + ), + ) + .add( + HttpApiEndpoint.delete("deleteConnector", "/connectors/:id", { + params: { id: GcpConnectorPublicId }, + success: V2GcpConnectorDeleteResponse, + error: [V2InsufficientPermissions.schema, connectorNotFound, integrationPersistence], + }).annotateMerge( + OpenApi.annotations({ + identifier: "deleteGcpConnector", + summary: "Disconnect a Google Cloud project", + description: + "Deletes the connector, after which Maple rejects the project's log pushes. Already-ingested data is unaffected. The response carries the cleanup script to run in the project. Requires an org-admin role and the `integrations:write` scope.", + }), + ), + ) + .prefix("/v2/integrations/gcp") + .middleware(AuthorizationV2) + .annotateMerge( + OpenApi.annotations({ + title: "Google Cloud Integration", + description: + "Connect Google Cloud projects to your organization: register a connector per project, fetch the `gcloud` scripts that route its logs to Maple, and see whether logs are arriving.", + }), + ) {} diff --git a/packages/domain/src/http/v2/openapi.test.ts b/packages/domain/src/http/v2/openapi.test.ts index cfd4aa936e..8bcc8c70df 100644 --- a/packages/domain/src/http/v2/openapi.test.ts +++ b/packages/domain/src/http/v2/openapi.test.ts @@ -94,6 +94,7 @@ describe("MapleApiV2 OpenAPI", () => { "DELETE /v2/dashboards/{id}/widgets/{widget_id}/share", "DELETE /v2/integrations/chat_connectors/{connector}/identity", "DELETE /v2/integrations/chat_workspaces/{id}", + "DELETE /v2/integrations/gcp/connectors/{id}", "DELETE /v2/integrations/planetscale", "DELETE /v2/mobile_devices/{token}", "DELETE /v2/mobile_devices/{token}/live_activities/{incident_id}", @@ -137,6 +138,7 @@ describe("MapleApiV2 OpenAPI", () => { "GET /v2/instrumentation/signals", "GET /v2/integrations/chat_connectors", "GET /v2/integrations/chat_workspaces/{id}/destinations", + "GET /v2/integrations/gcp", "GET /v2/integrations/planetscale", "GET /v2/integrations/planetscale/databases", "GET /v2/integrations/planetscale/organizations", @@ -194,6 +196,8 @@ describe("MapleApiV2 OpenAPI", () => { "POST /v2/instrumentation/recommendations/{id}/reopen", "POST /v2/integrations/chat_connectors/{connector}/identity/link", "POST /v2/integrations/chat_connectors/{connector}/install", + "POST /v2/integrations/gcp/connectors", + "POST /v2/integrations/gcp/connectors/{id}/setup_scripts", "POST /v2/integrations/planetscale/connect", "POST /v2/integrations/planetscale/events", "POST /v2/integrations/planetscale/metrics_token", diff --git a/packages/domain/src/http/v2/public-id.ts b/packages/domain/src/http/v2/public-id.ts index 06f31ff373..3640b119ef 100644 --- a/packages/domain/src/http/v2/public-id.ts +++ b/packages/domain/src/http/v2/public-id.ts @@ -41,6 +41,7 @@ export const PublicIdPrefixes = { mobileDevice: "mdev", chatWorkspace: "chatw", agentFeedback: "afb", + gcpConnector: "gcpc", /** Synthetic identity for logs, which have no native OTel record id. */ log: "log", /** Reserved for the future events/webhooks system. */ diff --git a/packages/primitives/src/index.ts b/packages/primitives/src/index.ts index f29791c0a9..cce44eb412 100644 --- a/packages/primitives/src/index.ts +++ b/packages/primitives/src/index.ts @@ -101,6 +101,16 @@ export type PostgresTransactionId = Schema.Schema.Type +export const GcpConnectorId = MapleUuidId("@maple/GcpConnectorId", "GCP Connector ID") +export type GcpConnectorId = Schema.Schema.Type + +// Google's project id rules. The id is interpolated into a shell script, so nothing looser. +export const GcpProjectId = Schema.String.check(Schema.isPattern(/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/u)).pipe( + Schema.brand("@maple/GcpProjectId"), + Schema.annotate({ identifier: "@maple/GcpProjectId", title: "Google Cloud Project ID" }), +) +export type GcpProjectId = Schema.Schema.Type + export const AlertDestinationId = MapleUuidId("@maple/AlertDestinationId", "Alert Destination ID") export type AlertDestinationId = Schema.Schema.Type From b2f50bdced5aa2ec7cceef77341d23f88c534d6c Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 17:50:56 +0200 Subject: [PATCH 02/21] fix(integrations): grant the sink Logs Writer, keep health transition logs, carry id brands Pre-PR review: the sink's writer identity also needs roles/logging.logWriter per the current routing docs; compute.googleapis.com/healthchecks only logs health state transitions, so it stays in the default filter's output while probe requests are still excluded; the id and project id columns carry their brands; service spans are annotated; the AAD binding is tested. --- .../routes/v2/integrations-gcp.http.test.ts | 7 ++-- .../integrations/GcpConnectorService.test.ts | 40 +++++++++++++++---- .../integrations/GcpConnectorService.ts | 16 ++++---- .../integrations/gcp/setup-scripts.test.ts | 18 +++++---- .../integrations/gcp/setup-scripts.ts | 20 +++++----- packages/db/src/schema/gcp-connectors.ts | 6 +-- packages/domain/src/gcp.test.ts | 12 ++---- packages/domain/src/gcp.ts | 10 ++--- .../domain/src/http/v2/integrations-gcp.ts | 23 +++++------ 9 files changed, 87 insertions(+), 65 deletions(-) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index 7484c8d600..ee521a1a9c 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -246,7 +246,7 @@ describe("v2 gcp integration over HTTP", () => { await harness.dispose() }) - it("rejects anything that is not a Google Cloud project id or a gcpc_ id before the service", async () => { + it("answers malformed project and connector ids with 400, an unknown connector with 404", async () => { const harness = makeHarness() const admin = await harness.adminKey() @@ -262,8 +262,9 @@ describe("v2 gcp integration over HTTP", () => { const { status } = await harness.request("DELETE", `${BASE}/connectors/${wrongPrefix}`, admin.secret) expect(status).toBe(400) - const empty = await harness.request("GET", BASE, admin.secret) - expect(empty.body.connectors).toEqual([]) + const unknown = encodePublicId("gcpc", "018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8") + const missing = await harness.request("DELETE", `${BASE}/connectors/${unknown}`, admin.secret) + expect(missing.status).toBe(404) await harness.dispose() }) }) diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts index 6086e13662..c7773f0623 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.test.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -4,7 +4,13 @@ import { hashIngestKey } from "@maple/db" import { gcpConnectorResourceNames } from "@maple/domain/gcp" import { GcpProjectId, OrgId, UserId } from "@maple/domain/primitives" import { Env } from "@maple/backend/platform/Env" -import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/platform/test-pglite" +import { + cleanupTestDbs, + createTestDb, + executeSql, + queryFirstRow, + type TestDb, +} from "@maple/backend/platform/test-pglite" import { GcpConnectorService, type GcpConnector } from "./GcpConnectorService" const trackedDbs: TestDb[] = [] @@ -92,11 +98,8 @@ describe("GcpConnectorService", () => { const duplicate = yield* Effect.flip(gcp.create(orgId, userId, projectId("acme-prod"))) assert.strictEqual(duplicate._tag, "@maple/http/errors/GcpProjectAlreadyConnectedError") - // Each org gets its own connector, and with it its own resource names in the project. - const theirs = yield* gcp.create(otherOrgId, userId, projectId("acme-prod")) - const ours = (yield* gcp.status(orgId)).connectors - assert.lengthOf(ours, 1) - assert.notDeepEqual(gcpConnectorResourceNames(theirs.id), gcpConnectorResourceNames(ours[0]!.id)) + yield* gcp.create(otherOrgId, userId, projectId("acme-prod")) + assert.lengthOf((yield* gcp.status(orgId)).connectors, 1) }).pipe(Effect.provide(makeLayer(testDb))) }) @@ -107,7 +110,8 @@ describe("GcpConnectorService", () => { const connector = yield* gcp.create(orgId, userId, projectId("acme-prod")) // What the ingest gateway writes after a rejected push. yield* Effect.promise(() => - testDb.pglite.query( + executeSql( + testDb, "UPDATE gcp_connectors SET last_received_at = $1, last_error = $2 WHERE id = $3", ["2026-10-08T09:12:00.000Z", "payload was not a LogEntry", connector.id], ), @@ -142,6 +146,28 @@ describe("GcpConnectorService", () => { ) }) + it.effect("refuses to render a secret that was moved onto another connector's row", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + const source = yield* gcp.create(orgId, userId, projectId("acme-prod")) + const target = yield* gcp.create(orgId, userId, projectId("acme-staging")) + yield* Effect.promise(() => + executeSql( + testDb, + `UPDATE gcp_connectors AS t + SET secret_ciphertext = s.secret_ciphertext, secret_iv = s.secret_iv, secret_tag = s.secret_tag + FROM gcp_connectors AS s WHERE s.id = $1 AND t.id = $2`, + [source.id, target.id], + ), + ) + const error = yield* Effect.flip( + gcp.scripts(orgId, target.id, { excludeGkeContainerLogs: false }), + ) + assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsPersistenceError") + }).pipe(Effect.provide(makeLayer(testDb))) + }) + it.effect("scopes scripts and deletion to the owning organization", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index bffbcbae6e..77bf711fc6 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -7,7 +7,7 @@ import { } from "@maple/domain/http" import { GcpConnectorId, type GcpProjectId, type OrgId, type UserId } from "@maple/domain/primitives" import { and, asc, eq } from "drizzle-orm" -import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { Clock, Context, Effect, Layer, Option, Redacted } from "effect" import { decryptAes256Gcm, encryptAes256Gcm, parseBase64Aes256GcmKey } from "@maple/backend/platform/Crypto" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" @@ -17,7 +17,7 @@ import { renderGcpCleanupScript, renderGcpSetupScript } from "./gcp/setup-script export interface GcpConnector { readonly id: GcpConnectorId - readonly projectId: string + readonly projectId: GcpProjectId readonly createdAt: number /** Last log push the ingest gateway accepted from this project. Null until the first one. */ readonly lastLogReceivedAt: number | null @@ -51,7 +51,7 @@ export interface GcpConnectorServiceApi { orgId: OrgId, connectorId: GcpConnectorId, ) => Effect.Effect< - { readonly projectId: string; readonly cleanupScript: string }, + { readonly projectId: GcpProjectId; readonly cleanupScript: string }, IntegrationsNotFoundError | IntegrationsPersistenceError > } @@ -61,13 +61,11 @@ const toPersistenceError = makePersistenceErrorMapper( "Google Cloud connector database error", ) -const decodeConnectorId = Schema.decodeUnknownSync(GcpConnectorId) - // Binds the ciphertext to its row, so a stored secret cannot be moved onto another connector. const secretAad = (connectorId: string) => Buffer.from(`gcp_connectors:v1:${connectorId}`, "utf8") const toConnector = (row: GcpConnectorRow): GcpConnector => ({ - id: decodeConnectorId(row.id), + id: row.id, projectId: row.projectId, createdAt: dateToMs(row.createdAt), lastLogReceivedAt: dateToMs(row.lastReceivedAt), @@ -99,6 +97,7 @@ export class GcpConnectorService extends Context.Service db .select() @@ -117,7 +116,8 @@ export class GcpConnectorService extends Context.Service db .select() @@ -193,6 +194,7 @@ export class GcpConnectorService extends Context.Service db .delete(gcpConnectors) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index 259650e317..dd0f7b9463 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { afterEach, describe, expect, it } from "vitest" import { Schema } from "effect" -import { GcpConnectorId } from "@maple/domain/primitives" +import { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" import { gcpLogFilter, renderGcpCleanupScript, @@ -13,6 +13,7 @@ import { } from "./setup-scripts" const connectorId = Schema.decodeUnknownSync(GcpConnectorId)("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8") +const projectId = Schema.decodeUnknownSync(GcpProjectId)("acme-prod") const NAME = "maple-018f2b3c4d5e4f708192a3b4" const SECRET = "maple_gcp_s3cr3t-value" const WRITER = "serviceAccount:service-1@gcp-sa-logging.iam.gserviceaccount.com" @@ -20,7 +21,7 @@ const MAPLE_ACCOUNT = "collector@maple-prod.iam.gserviceaccount.com" const input: GcpSetupScriptInput = { connectorId, - projectId: "acme-prod", + projectId, pushEndpoint: `https://ingest.test/v1/logpush/gcp/${connectorId}?secret=${SECRET}`, mapleServiceAccountEmail: MAPLE_ACCOUNT, excludeGkeContainerLogs: false, @@ -83,6 +84,7 @@ describe("renderGcpSetupScript", () => { `logging sinks describe ${NAME} --project=acme-prod`, `logging sinks create ${NAME} pubsub.googleapis.com/projects/acme-prod/topics/${NAME} --log-filter=${gcpLogFilter(false)} --project=acme-prod`, `logging sinks describe ${NAME} --project=acme-prod --format=value(writerIdentity)`, + `projects add-iam-policy-binding acme-prod --member=${WRITER} --role=roles/logging.logWriter --condition=None`, `pubsub topics add-iam-policy-binding ${NAME} --project=acme-prod --member=${WRITER} --role=roles/pubsub.publisher`, `iam service-accounts describe ${NAME}@acme-prod.iam.gserviceaccount.com --project=acme-prod`, `iam service-accounts create ${NAME} --project=acme-prod --display-name=Maple metrics reader`, @@ -115,9 +117,9 @@ describe("renderGcpSetupScript", () => { expect(commands.at(-1)).toContain("pubsub topics add-iam-policy-binding") }) - it("excludes data-access audit logs and health checks by default, GKE containers on request", () => { + it("excludes data-access audit logs and health-check probes by default, GKE containers on request", () => { expect(gcpLogFilter(false)).toBe( - 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT log_id("compute.googleapis.com/healthchecks") AND NOT httpRequest.userAgent:"GoogleHC"', + 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC"', ) expect(gcpLogFilter(true)).toBe(`${gcpLogFilter(false)} AND NOT resource.type="k8s_container"`) expect(renderGcpSetupScript({ ...input, excludeGkeContainerLogs: true })).toContain( @@ -127,15 +129,17 @@ describe("renderGcpSetupScript", () => { it("passes interpolated values to gcloud verbatim, whatever they contain", () => { const hostile = `x'; touch pwned; echo '$(touch pwned)` - const { dir, status, commands } = run(renderGcpSetupScript({ ...input, projectId: hostile })) + const { dir, status, commands } = run( + renderGcpSetupScript({ ...input, mapleServiceAccountEmail: hostile }), + ) expect(status).toBe(0) - expect(commands[0]).toContain(`--project=${hostile}`) + expect(commands.at(-1)).toContain(`--member=serviceAccount:${hostile} `) expect(existsSync(join(dir, "pwned"))).toBe(false) }) }) describe("renderGcpCleanupScript", () => { - const script = renderGcpCleanupScript(connectorId, "acme-prod") + const script = renderGcpCleanupScript(connectorId, projectId) it("carries the secret in the setup script only", () => { expect(renderGcpSetupScript(input)).toContain(SECRET) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index d518fc9465..4081551640 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -1,14 +1,13 @@ import { gcpConnectorResourceNames } from "@maple/domain/gcp" -import type { GcpConnectorId } from "@maple/domain/primitives" +import type { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" /** Single-quote a value for bash. Every value that reaches a script goes through here. */ const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` -// Data Access audit logs record every API read, and health-check probes repeat every few -// seconds per backend: both are high volume and say little about the workload. +// Data Access audit logs record every API read, and load balancer health-check probes hit each +// backend every few seconds: both are high volume and say little about the workload. const DEFAULT_LOG_FILTER = [ 'NOT log_id("cloudaudit.googleapis.com/data_access")', - 'NOT log_id("compute.googleapis.com/healthchecks")', 'NOT httpRequest.userAgent:"GoogleHC"', ] @@ -17,7 +16,7 @@ export const gcpLogFilter = (excludeGkeContainerLogs: boolean): string => " AND ", ) -const logVariables = (connectorId: GcpConnectorId, projectId: string): string => { +const logVariables = (connectorId: GcpConnectorId, projectId: GcpProjectId): string => { const names = gcpConnectorResourceNames(connectorId) return `PROJECT_ID=${sh(projectId)} TOPIC=${sh(names.topic)} @@ -58,7 +57,7 @@ retry gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL export interface GcpSetupScriptInput { readonly connectorId: GcpConnectorId - readonly projectId: string + readonly projectId: GcpProjectId /** The ingest gateway's receiver URL for this connector, including its secret. */ readonly pushEndpoint: string /** Maple's own Google service account. Undefined renders a logs-only script. */ @@ -118,8 +117,10 @@ else gcloud logging sinks create "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" --project="$PROJECT_ID" fi -# The sink writes as its own Google-managed identity, which needs to publish to the topic. +# The sink writes as a Google-managed identity, which needs to route logs and publish to the topic. WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" --project="$PROJECT_ID" --format='value(writerIdentity)')" +gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ + --member="$WRITER_IDENTITY" --role=roles/logging.logWriter --condition=None >/dev/null gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher >/dev/null ${mapleAccount === undefined ? "" : metricsSteps(input.connectorId, mapleAccount)} @@ -128,11 +129,12 @@ echo "Maple setup complete for $PROJECT_ID. Logs start arriving in Maple within } /** Removes what the setup script created. Carries no secret, so it outlives the connector. */ -export const renderGcpCleanupScript = (connectorId: GcpConnectorId, projectId: string): string => +export const renderGcpCleanupScript = (connectorId: GcpConnectorId, projectId: GcpProjectId): string => `#!/usr/bin/env bash # Maple: remove what the Maple setup script created in this Google Cloud project. # Run in Cloud Shell as a project owner. No "set -e": every step runs, and a -# NOT_FOUND error only means that resource is already gone. +# NOT_FOUND error only means that resource is already gone. The logging service +# agent keeps its Logs Writer role: every sink in the project shares that identity. set -uo pipefail ${logVariables(connectorId, projectId)} diff --git a/packages/db/src/schema/gcp-connectors.ts b/packages/db/src/schema/gcp-connectors.ts index adc91d8dd8..35cc69730f 100644 --- a/packages/db/src/schema/gcp-connectors.ts +++ b/packages/db/src/schema/gcp-connectors.ts @@ -1,4 +1,4 @@ -import type { OrgId } from "@maple/domain" +import type { GcpConnectorId, GcpProjectId, OrgId } from "@maple/domain" import { pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" // One row per connected Google Cloud project. The ingest gateway's `/v1/logpush/gcp/...` receiver @@ -6,9 +6,9 @@ import { pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" export const gcpConnectors = pgTable( "gcp_connectors", { - id: text("id").notNull().primaryKey(), + id: text("id").$type().notNull().primaryKey(), orgId: text("org_id").$type().notNull(), - projectId: text("project_id").notNull(), + projectId: text("project_id").$type().notNull(), secretCiphertext: text("secret_ciphertext").notNull(), secretIv: text("secret_iv").notNull(), secretTag: text("secret_tag").notNull(), diff --git a/packages/domain/src/gcp.test.ts b/packages/domain/src/gcp.test.ts index 54419e700a..502ea6aa0f 100644 --- a/packages/domain/src/gcp.test.ts +++ b/packages/domain/src/gcp.test.ts @@ -3,11 +3,11 @@ import { Schema } from "effect" import { gcpConnectorResourceNames } from "./gcp" import { GcpConnectorId } from "./primitives" -const connectorId = Schema.decodeUnknownSync(GcpConnectorId) - describe("gcpConnectorResourceNames", () => { it("derives a service account id inside Google's limits from the connector id alone", () => { - const names = gcpConnectorResourceNames(connectorId("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8")) + const names = gcpConnectorResourceNames( + Schema.decodeUnknownSync(GcpConnectorId)("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8"), + ) expect(names).toEqual({ serviceAccountId: "maple-018f2b3c4d5e4f708192a3b4", topic: "maple-018f2b3c4d5e4f708192a3b4", @@ -17,10 +17,4 @@ describe("gcpConnectorResourceNames", () => { // Google: 6-30 characters, [a-z]([-a-z0-9]*[a-z0-9]). expect(names.serviceAccountId).toMatch(/^[a-z][-a-z0-9]{4,28}[a-z0-9]$/) }) - - it("gives two connectors different names", () => { - expect(gcpConnectorResourceNames(connectorId("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8"))).not.toEqual( - gcpConnectorResourceNames(connectorId("118f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8")), - ) - }) }) diff --git a/packages/domain/src/gcp.ts b/packages/domain/src/gcp.ts index 8404de0012..0eee9d3a69 100644 --- a/packages/domain/src/gcp.ts +++ b/packages/domain/src/gcp.ts @@ -1,12 +1,10 @@ import type { GcpConnectorId } from "./primitives" /** - * Names of the resources a connector's setup script creates in the customer's Google Cloud - * project. Derived from the connector id alone, so nothing extra is stored and the names cannot - * be chosen by a caller: the metrics reader of one connector is never the account another - * organization's connector impersonates. - * - * 24 hex characters of the id keep the service account id at Google's 30-character limit. + * Names of what a connector's setup script creates in the customer's Google Cloud project. + * Derived from the connector id alone, so a caller can never pick them: one connector's metrics + * account is not the one another organization's connector impersonates. 24 hex characters keep + * the service account id at Google's 30-character limit. */ export const gcpConnectorResourceNames = (connectorId: GcpConnectorId) => { const name = `maple-${connectorId.replaceAll("-", "").slice(0, 24)}` diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts index 374b956166..813711324a 100644 --- a/packages/domain/src/http/v2/integrations-gcp.ts +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -12,14 +12,9 @@ import { V2InsufficientPermissions } from "./errors" import { publicErrors } from "./public-error" import { PublicId, PublicIdPrefixes } from "./public-id" -// Google Cloud integration. There is no OAuth: an org registers a connector for one Google Cloud -// project and its owner runs the generated `gcloud` script in Cloud Shell. The script routes the -// project's logs through Pub/Sub to Maple's ingest gateway and, where this deployment has a Google -// identity, creates a read-only metrics account Maple may impersonate. -// -// Public v2 from the start: infrastructure-as-code setups create connectors with a scoped API key, -// which only v2 has (docs/api-v2.md). The log receiver itself is a provider-facing route on the -// ingest gateway, not part of this group. +// Google Cloud integration. No OAuth: an org registers a connector per Google Cloud project and +// its owner runs the generated `gcloud` script, which routes the project's logs through Pub/Sub +// to the ingest gateway. Public v2 from the start so scripted setups can use a scoped API key. /** `gcpc_…` public ID ⇄ internal `GcpConnectorId` (raw UUID). */ export const GcpConnectorPublicId = PublicId(PublicIdPrefixes.gcpConnector, GcpConnectorId) @@ -116,11 +111,11 @@ export const V2GcpSetupScripts = Schema.Struct({ setup_script: Schema.String.annotate({ description: "A bash script for Cloud Shell, run by a project owner. It embeds the connector's push secret, so treat it as a credential. Safe to re-run; the log filter is an editable variable at the top.", - examples: ["#!/usr/bin/env bash\nset -euo pipefail\n…"], + examples: ["#!/usr/bin/env bash\n…"], }), cleanup_script: Schema.String.annotate({ description: "A bash script that removes everything the setup script created. Carries no secret.", - examples: ["#!/usr/bin/env bash\nset -euo pipefail\n…"], + examples: ["#!/usr/bin/env bash\n…"], }), }).annotate({ identifier: "GcpSetupScripts", @@ -129,8 +124,8 @@ export const V2GcpSetupScripts = Schema.Struct({ examples: [ wireExample({ object: "gcp_connector.setup_scripts", - setup_script: "#!/usr/bin/env bash\nset -euo pipefail\n…", - cleanup_script: "#!/usr/bin/env bash\nset -euo pipefail\n…", + setup_script: "#!/usr/bin/env bash\n…", + cleanup_script: "#!/usr/bin/env bash\n…", }), ], }) @@ -147,7 +142,7 @@ export const V2GcpConnectorDeleteResponse = Schema.Struct({ cleanup_script: Schema.String.annotate({ description: "A bash script that removes the resources the setup script created. Maple has no write access to the project, so the project owner runs it.", - examples: ["#!/usr/bin/env bash\nset -euo pipefail\n…"], + examples: ["#!/usr/bin/env bash\n…"], }), }).annotate({ identifier: "GcpConnectorDeleteResponse", @@ -158,7 +153,7 @@ export const V2GcpConnectorDeleteResponse = Schema.Struct({ id: connectorExample.id, object: "gcp_connector", deleted: true, - cleanup_script: "#!/usr/bin/env bash\nset -euo pipefail\n…", + cleanup_script: "#!/usr/bin/env bash\n…", }), ], }) From a2826df374bb197c95edefc98df5f247649dad18 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 18:03:53 +0200 Subject: [PATCH 03/21] chore(ios): regenerate the OpenAPI spec for the Google Cloud tag --- apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index d51844013e..c1098e28ce 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -7737,6 +7737,10 @@ "description": "Connect PlanetScale to your organization and manage what Maple collects from it: connection status, organization binding, the metrics service token that enables branch-metrics scraping, the database inventory, webhook setup, query insights, and the lifecycle event timeline.", "name": "PlanetScale Integration" }, + { + "description": "Connect Google Cloud projects to your organization: register a connector per project, fetch the `gcloud` scripts that route its logs to Maple, and see whether logs are arriving.", + "name": "Google Cloud Integration" + }, { "description": "Deduplicated errors and alert-backed issues tracked through Maple's triage workflow.", "name": "Error Issues" From bc7d03bd5197ede9b5c895cf376c982776061ffc Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 19:29:27 +0200 Subject: [PATCH 04/21] feat(integrations): scope Google Cloud connectors to a project, folder or organization A connector now names a scope and a host project, and opts into log forwarding and into metrics plus resource collection separately (PATCH /connectors/{id}). The setup script places the sink and the read-only role bindings at the scope and converges: it removes what a disabled capability left behind. The create-table migration is regenerated; nothing was deployed on the old one. --- .../routes/v2/integrations-gcp.http.test.ts | 205 +++++++++---- .../src/routes/v2/integrations-gcp.http.ts | 43 ++- .../MapleAPI/Sources/MapleAPI/openapi.json | 2 +- docs/api-v2.md | 60 ++-- .../src/services/audit/audit-actions.ts | 4 +- .../integrations/GcpConnectorService.test.ts | 159 ++++++---- .../integrations/GcpConnectorService.ts | 184 +++++++++--- .../integrations/gcp/setup-scripts.test.ts | 225 +++++++++----- .../integrations/gcp/setup-scripts.ts | 279 ++++++++++++------ .../migration.sql | 6 +- .../snapshot.json | 65 +++- packages/db/src/schema/gcp-connectors.ts | 17 +- .../anticipated-error-identifiers.ts | 3 +- packages/domain/src/http/integrations.ts | 41 ++- .../domain/src/http/v2/integrations-gcp.ts | 187 +++++++++--- packages/domain/src/http/v2/openapi.test.ts | 1 + packages/primitives/src/index.ts | 11 + 17 files changed, 1098 insertions(+), 394 deletions(-) rename packages/db/drizzle/{20261008152420_gcp_connectors => 20261008171958_gcp_connectors}/migration.sql (65%) rename packages/db/drizzle/{20261008152420_gcp_connectors => 20261008171958_gcp_connectors}/snapshot.json (99%) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index ee521a1a9c..6b620097b0 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -24,7 +24,8 @@ import { /** * `/v2/integrations/gcp` over an embedded PGlite with the real connector service: the wire shape - * a script parses, the admin gate, and who can reach the push secret. + * a script parses, the scope and capability rules, the admin gate, and who can reach the push + * secret. */ const createdDbs: TestDb[] = [] @@ -34,7 +35,7 @@ const ORG = Schema.decodeUnknownSync(OrgId)("org_gcp_e2e") const USER = Schema.decodeUnknownSync(UserId)("user_gcp_e2e") const BASE = "/v2/integrations/gcp" -const testConfig = () => +const testConfig = (metricsAvailable: boolean) => ConfigProvider.layer( ConfigProvider.fromUnknown({ PORT: "3478", @@ -48,12 +49,15 @@ const testConfig = () => MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", MAPLE_INGEST_PUBLIC_URL: "https://ingest.test", INTERNAL_SERVICE_TOKEN: "test-internal-token", + ...(metricsAvailable + ? { MAPLE_GCP_SERVICE_ACCOUNT_EMAIL: "collector@maple-prod.iam.gserviceaccount.com" } + : undefined), }), ) -const makeHarness = () => { +const makeHarness = (options: { readonly metricsAvailable: boolean } = { metricsAvailable: true }) => { const testDb = createTestDb(createdDbs) - const envLive = Env.layer.pipe(Layer.provide(testConfig())) + const envLive = Env.layer.pipe(Layer.provide(testConfig(options.metricsAvailable))) const servicesLive = Layer.mergeAll( ApiKeysService.layer, AuthService.layer, @@ -123,41 +127,62 @@ const makeHarness = () => { } } +const CONNECTORS = `${BASE}/connectors` +const organizationConnector = { + scope_type: "organization", + scope_id: "123456789012", + project_id: "acme-host", + metrics_enabled: true, +} +const projectConnector = { scope_type: "project", scope_id: "acme-prod" } + describe("v2 gcp integration over HTTP", () => { - it("creates a connector, lists it for any member, and deletes it with a cleanup script", async () => { + it("creates connectors per scope, lists them for any member, deletes with a cleanup script", async () => { const harness = makeHarness() const admin = await harness.adminKey() const member = await harness.memberKey() - const created = await harness.request("POST", `${BASE}/connectors`, admin.secret, { - project_id: "acme-prod", - }) - expect(created.status).toBe(200) - expect(created.body).toEqual({ + const organization = await harness.request("POST", CONNECTORS, admin.secret, organizationConnector) + expect(organization.status).toBe(200) + expect(organization.body).toEqual({ id: expect.stringMatching(/^gcpc_/), object: "gcp_connector", - project_id: "acme-prod", + scope_type: "organization", + scope_id: "123456789012", + project_id: "acme-host", + logs_enabled: true, + metrics_enabled: true, created_at: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T.*Z$/), last_log_received_at: null, last_log_error: null, }) + // A project is its own host project, and forwards logs only unless asked otherwise. + const project = await harness.request("POST", CONNECTORS, admin.secret, projectConnector) + expect(project.body).toMatchObject({ + scope_type: "project", + scope_id: "acme-prod", + project_id: "acme-prod", + logs_enabled: true, + metrics_enabled: false, + }) + const status = await harness.request("GET", BASE, member.secret) expect(status.status).toBe(200) - expect(status.body).toEqual({ - object: "gcp_integration", - metrics_available: false, - connectors: [created.body], - }) + expect(status.body).toMatchObject({ object: "gcp_integration", metrics_available: true }) + expect(status.body.connectors).toEqual(expect.arrayContaining([organization.body, project.body])) + expect(status.body.connectors).toHaveLength(2) - const path = `${BASE}/connectors/${created.body.id}` + const path = `${CONNECTORS}/${organization.body.id}` const deleted = await harness.request("DELETE", path, admin.secret) expect(deleted.status).toBe(200) expect(deleted.body).toEqual({ - id: created.body.id, + id: organization.body.id, object: "gcp_connector", deleted: true, - cleanup_script: expect.stringContaining("gcloud logging sinks delete"), + cleanup_script: expect.stringContaining( + 'gcloud logging sinks delete "$SINK" --organization="$SCOPE_ID"', + ), }) const gone = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) @@ -169,29 +194,72 @@ describe("v2 gcp integration over HTTP", () => { await harness.dispose() }) - it("renders the scripts for an admin, with the push secret only in the setup script", async () => { + it("renders scripts that follow the connector's scope and switches", async () => { const harness = makeHarness() const admin = await harness.adminKey() - const created = await harness.request("POST", `${BASE}/connectors`, admin.secret, { - project_id: "acme-prod", + const created = await harness.request("POST", CONNECTORS, admin.secret, { + ...organizationConnector, + scope_type: "folder", }) - const path = `${BASE}/connectors/${created.body.id}/setup_scripts` + const path = `${CONNECTORS}/${created.body.id}` - const { status, body } = await harness.request("POST", path, admin.secret, { + const both = await harness.request("POST", `${path}/setup_scripts`, admin.secret, { exclude_gke_container_logs: true, }) - expect(status).toBe(200) - expect(Object.keys(body).sort()).toEqual(["cleanup_script", "object", "setup_script"]) - expect(body.object).toBe("gcp_connector.setup_scripts") - expect(body.setup_script).toMatch( + expect(both.status).toBe(200) + expect(Object.keys(both.body).sort()).toEqual(["cleanup_script", "object", "setup_script"]) + expect(both.body.object).toBe("gcp_connector.setup_scripts") + expect(both.body.setup_script).toMatch( /PUSH_ENDPOINT='https:\/\/ingest\.test\/v1\/logpush\/gcp\/[0-9a-f-]{36}\?secret=maple_gcp_/, ) - expect(body.setup_script).toContain("PROJECT_ID='acme-prod'") - expect(body.setup_script).toContain('NOT resource.type="k8s_container"') - expect(body.cleanup_script).not.toContain("maple_gcp_") + expect(both.body.setup_script).toContain("PROJECT_ID='acme-host'") + expect(both.body.setup_script).toContain("SCOPE_ID='123456789012'") + expect(both.body.setup_script).toContain('--folder="$SCOPE_ID" --include-children') + expect(both.body.setup_script).toContain('NOT resource.type="k8s_container"') + expect(both.body.setup_script).toContain( + 'gcloud resource-manager folders add-iam-policy-binding "$SCOPE_ID"', + ) + expect(both.body.cleanup_script).not.toContain("maple_gcp_") + + const patched = await harness.request("PATCH", path, admin.secret, { logs_enabled: false }) + expect(patched.status).toBe(200) + expect(patched.body).toEqual({ ...created.body, logs_enabled: false }) + + const metricsOnly = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) + expect(metricsOnly.body.setup_script).not.toContain("maple_gcp_") + expect(metricsOnly.body.setup_script).toContain("gcloud logging sinks delete") + await harness.dispose() + }) + + it("keeps one capability on, and refuses metrics on a deployment without a Google identity", async () => { + const harness = makeHarness({ metricsAvailable: false }) + const admin = await harness.adminKey() + + const unavailable = await harness.request("POST", CONNECTORS, admin.secret, organizationConnector) + expect(unavailable.status).toBe(409) + expect(unavailable.body.error).toMatchObject({ + _tag: "@maple/http/errors/GcpMetricsUnavailableError", + code: "gcp_metrics_unavailable", + param: "metrics_enabled", + }) + + const nothing = await harness.request("POST", CONNECTORS, admin.secret, { + ...projectConnector, + logs_enabled: false, + }) + expect(nothing.status).toBe(400) + expect(nothing.body.error.code).toBe("integration_request_invalid") - const defaults = await harness.request("POST", path, admin.secret, {}) - expect(defaults.body.setup_script).not.toContain("k8s_container") + const created = await harness.request("POST", CONNECTORS, admin.secret, projectConnector) + const path = `${CONNECTORS}/${created.body.id}` + expect((await harness.request("GET", BASE, admin.secret)).body.metrics_available).toBe(false) + + const turnOn = await harness.request("PATCH", path, admin.secret, { metrics_enabled: true }) + expect(turnOn.status).toBe(409) + expect(turnOn.body.error.code).toBe("gcp_metrics_unavailable") + const turnOff = await harness.request("PATCH", path, admin.secret, { logs_enabled: false }) + expect(turnOff.status).toBe(400) + expect(turnOff.body.error.code).toBe("integration_request_invalid") await harness.dispose() }) @@ -200,71 +268,88 @@ describe("v2 gcp integration over HTTP", () => { const admin = await harness.adminKey() const member = await harness.memberKey() const readOnly = await harness.adminKey(["integrations:read"]) - const created = await harness.request("POST", `${BASE}/connectors`, admin.secret, { - project_id: "acme-prod", - }) - const path = `${BASE}/connectors/${created.body.id}` + const created = await harness.request("POST", CONNECTORS, admin.secret, projectConnector) + const path = `${CONNECTORS}/${created.body.id}` for (const [method, target, body] of [ - ["POST", `${BASE}/connectors`, { project_id: "other-project" }], + ["POST", CONNECTORS, { scope_type: "project", scope_id: "other-project" }], + ["PATCH", path, { metrics_enabled: true }], ["POST", `${path}/setup_scripts`, {}], ["DELETE", path, undefined], ] as const) { const asMember = await harness.request(method, target, member.secret, body) - expect(asMember.status).toBe(403) + expect(asMember.status, `${method} as member`).toBe(403) expect(asMember.body.error).toMatchObject({ type: "permission_error", code: "insufficient_permissions", }) const asReadOnly = await harness.request(method, target, readOnly.secret, body) - expect(asReadOnly.status).toBe(403) + expect(asReadOnly.status, `${method} with a read-only key`).toBe(403) expect(asReadOnly.body.error.code).toBe("insufficient_scope") } const status = await harness.request("GET", BASE, readOnly.secret) expect(status.status).toBe(200) - expect(status.body.connectors).toHaveLength(1) + expect(status.body.connectors).toEqual([created.body]) expect(JSON.stringify(status.body)).not.toContain("maple_gcp_") await harness.dispose() }) - it("answers a project that is already connected with a 409 naming the field", async () => { + it("answers a scope that is already connected with a 409 naming the field", async () => { const harness = makeHarness() const admin = await harness.adminKey() - await harness.request("POST", `${BASE}/connectors`, admin.secret, { project_id: "acme-prod" }) + await harness.request("POST", CONNECTORS, admin.secret, organizationConnector) - const { status, body } = await harness.request("POST", `${BASE}/connectors`, admin.secret, { - project_id: "acme-prod", + const { status, body } = await harness.request("POST", CONNECTORS, admin.secret, { + ...organizationConnector, + project_id: "another-host", }) expect(status).toBe(409) expect(body.error).toMatchObject({ - _tag: "@maple/http/errors/GcpProjectAlreadyConnectedError", + _tag: "@maple/http/errors/GcpScopeAlreadyConnectedError", type: "conflict_error", - code: "gcp_project_already_connected", - param: "project_id", + code: "gcp_scope_already_connected", + param: "scope_id", }) await harness.dispose() }) - it("answers malformed project and connector ids with 400, an unknown connector with 404", async () => { + it("answers malformed scopes and connector ids with 400, an unknown connector with 404", async () => { const harness = makeHarness() const admin = await harness.adminKey() - for (const projectId of ["acme'; rm -rf /; '", "Acme-Prod", "short", "acme-prod-", "$(id)-project"]) { - const { status, body } = await harness.request("POST", `${BASE}/connectors`, admin.secret, { - project_id: projectId, - }) - expect(status, projectId).toBe(400) - expect(body.error.type).toBe("invalid_request_error") + for (const body of [ + { scope_type: "project", scope_id: "acme'; echo pwned; '" }, + { scope_type: "project", scope_id: "Acme-Prod" }, + { scope_type: "project", scope_id: "$(id)-project" }, + // A project hosts itself. + { scope_type: "project", scope_id: "acme-prod", project_id: "acme-host" }, + // Folders and organizations are numbers and need a host project. + { scope_type: "organization", scope_id: "acme.example", project_id: "acme-host" }, + { scope_type: "folder", scope_id: "123456789012" }, + { scope_type: "folder", scope_id: "123456789012", project_id: "123456789012" }, + { scope_type: "billing_account", scope_id: "123456789012", project_id: "acme-host" }, + { scope_id: "acme-prod" }, + ]) { + const response = await harness.request("POST", CONNECTORS, admin.secret, body) + expect(response.status, JSON.stringify(body)).toBe(400) + expect(response.body.error.type).toBe("invalid_request_error") } + // The accepted spelling of the rejected project case above. + const selfHosted = await harness.request("POST", CONNECTORS, admin.secret, { + ...projectConnector, + project_id: "acme-prod", + }) + expect(selfHosted.status).toBe(200) const wrongPrefix = encodePublicId("scrp", "018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8") - const { status } = await harness.request("DELETE", `${BASE}/connectors/${wrongPrefix}`, admin.secret) - expect(status).toBe(400) + const malformed = await harness.request("DELETE", `${CONNECTORS}/${wrongPrefix}`, admin.secret) + expect(malformed.status).toBe(400) - const unknown = encodePublicId("gcpc", "018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8") - const missing = await harness.request("DELETE", `${BASE}/connectors/${unknown}`, admin.secret) - expect(missing.status).toBe(404) + const unknown = `${CONNECTORS}/${encodePublicId("gcpc", "018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8")}` + expect((await harness.request("DELETE", unknown, admin.secret)).status).toBe(404) + const patch = await harness.request("PATCH", unknown, admin.secret, { logs_enabled: true }) + expect(patch.status).toBe(404) await harness.dispose() }) }) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.ts b/apps/api/src/routes/v2/integrations-gcp.http.ts index f2de5f42ce..b029ce1b5d 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.ts @@ -16,18 +16,31 @@ import { } from "@maple/backend/services/integrations/GcpConnectorService" // Reading the status is open to every member. Everything else is admin-only: a connector decides -// which project may write logs into the org, and its setup script carries the push secret. +// what may write logs into the org and what Maple reads, and its setup script carries the push +// secret. const adminOnly = () => V2InsufficientPermissions.make("Only org admins can manage Google Cloud connectors") const toV2Connector = (connector: GcpConnector): V2GcpConnector => ({ id: connector.id, object: "gcp_connector", + scope_type: connector.scopeType, + scope_id: connector.scopeId, project_id: connector.projectId, + logs_enabled: connector.logsEnabled, + metrics_enabled: connector.metricsEnabled, created_at: isoTimestamp(connector.createdAt), last_log_received_at: isoTimestampOrNull(connector.lastLogReceivedAt), last_log_error: connector.lastLogError, }) +const auditMetadata = (connector: GcpConnector) => ({ + scope_type: connector.scopeType, + scope_id: connector.scopeId, + project_id: connector.projectId, + logs_enabled: connector.logsEnabled, + metrics_enabled: connector.metricsEnabled, +}) + export const HttpV2GcpIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "gcpIntegration", (handlers) => Effect.gen(function* () { const gcp = yield* GcpConnectorService @@ -48,10 +61,32 @@ export const HttpV2GcpIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "gcpIn Effect.gen(function* () { const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly) - const connector = yield* gcp.create(tenant.orgId, tenant.userId, payload.project_id) + const connector = yield* gcp.create(tenant.orgId, tenant.userId, { + scopeType: payload.scope_type, + scopeId: payload.scope_id, + // A project hosts its own resources; the contract rejects any other value. + projectId: payload.scope_type === "project" ? payload.scope_id : payload.project_id, + logsEnabled: payload.logs_enabled ?? true, + metricsEnabled: payload.metrics_enabled ?? false, + }) yield* recordHttpAudit("gcp_connector.created", { resourceId: connector.id, - metadata: { project_id: connector.projectId }, + metadata: auditMetadata(connector), + }) + return toV2Connector(connector) + }), + ) + .handle("updateConnector", ({ params, payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles, adminOnly) + const connector = yield* gcp.update(tenant.orgId, params.id, { + logsEnabled: payload.logs_enabled, + metricsEnabled: payload.metrics_enabled, + }) + yield* recordHttpAudit("gcp_connector.updated", { + resourceId: connector.id, + metadata: auditMetadata(connector), }) return toV2Connector(connector) }), @@ -77,7 +112,7 @@ export const HttpV2GcpIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "gcpIn const deleted = yield* gcp.delete(tenant.orgId, params.id) yield* recordHttpAudit("gcp_connector.deleted", { resourceId: params.id, - metadata: { project_id: deleted.projectId }, + metadata: auditMetadata(deleted.connector), }) return { id: params.id, diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index c1098e28ce..7a77b08209 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -7738,7 +7738,7 @@ "name": "PlanetScale Integration" }, { - "description": "Connect Google Cloud projects to your organization: register a connector per project, fetch the `gcloud` scripts that route its logs to Maple, and see whether logs are arriving.", + "description": "Connect Google Cloud projects, folders or organizations: register a connector, choose whether it forwards logs and whether Maple may read metrics and resources, fetch the `gcloud` scripts that set it up, and see whether logs are arriving.", "name": "Google Cloud Integration" }, { diff --git a/docs/api-v2.md b/docs/api-v2.md index bd8c825e42..43679ea574 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -27,7 +27,7 @@ PlanetScale was the first promotion under that rule, and it shows what "a public Promotion does not imply deletion. PlanetScale's v1 endpoints stay mounted and are marked `deprecated` in the v1 OpenAPI: the dashboard is off them, so nothing in this repo would notice if they broke, but customers may still be calling them. The general rule above ("each v1 group is deleted once nothing consumes it") means _nothing_, including callers outside this repo, so a promoted provider's v1 surface is removed only once its access logs go quiet. Until then both surfaces are live over the same services, and only v2 gets new work. -Google Cloud started on v2 for the same reason: there is no OAuth, so the whole setup is scriptable. A caller creates a connector with a scoped key and runs the returned `gcloud` script in the project. `status` is ungated; creating and deleting connectors and `setup_scripts` (it carries the push secret, and is a POST so a read-scoped key cannot fetch it) require org-admin. +Google Cloud started on v2 for the same reason: there is no OAuth, so the whole setup is scriptable. A caller creates a connector for a project, a folder or an organization with a scoped key, chooses whether it forwards logs and whether Maple may read metrics and resources, and runs the returned `gcloud` script. `status` is ungated; creating, updating and deleting connectors and `setup_scripts` (it carries the push secret, and is a POST so a read-scoped key cannot fetch it) require org-admin. Cloudflare, GitHub, and Hazel stay on the v1 `integrations` group (`/api/integrations`) with no v2 counterpart. They predate v2 and nothing public depends on them. Per the rule above, each gets promoted individually if something does. Scope families are derived mechanically from the first path segment under `/v2`, so every provider mounted at `/v2/integrations/` shares the `integrations:read` / `integrations:write` family. @@ -155,35 +155,35 @@ Stripe-style `expand[]` is deliberately omitted: responses embed the small, alwa Implemented in phases; the pilot (`api_keys`) ships first and proves every convention. -| Resource | Endpoints | Backing service | -| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| `api_keys` ✅ pilot | list/create/retrieve/roll/revoke, `scopes` param | `ApiKeysService` | -| `ingest_keys` ✅ | retrieve, `POST …/public/roll`, `POST …/private/roll` | `OrgIngestKeysService` | -| `dashboards` ✅ | CRUD + `versions` (list/retrieve/restore) + `templates` (list/preview/instantiate) + Perses import | `dashboards` | -| `alerts/rules` ✅ | CRUD + `test` + `preview` + `checks` | `AlertsService` | -| `alerts/destinations` ✅ | CRUD + `test` | `AlertsService` | -| `alerts/incidents` ✅ | list/retrieve | `AlertsService` | -| `alerts/deliveries` ✅ | list delivery attempts | `AlertsService` | -| `error_issues` 🟡 | list/retrieve/`service_counts` ✅; `events`, `comments`, `transitions`, `assignee`, `severity` deferred | `errors` | -| `investigations` ✅ | list/retrieve/create/status | `InvestigationService` | -| `anomalies` ✅ | incidents list/retrieve/timeseries/resolve/link-issue, `service_counts` aggregate + `PATCH` settings | `anomalies` | -| `instrumentation/recommendations` ✅ | list + dismiss/reopen | `RecommendationIssueService` | -| `instrumentation/audit` ✅ | retrieve (singleton report, recomputed per request) | `SetupAuditService` | -| `onboarding/checklist` ✅ | retrieve + claim (singleton; steps verified per request, claim admin-only and idempotent) | `OnboardingChecklistService` | -| `scrape_targets` ✅ | CRUD + `probe` + `checks` | `ScrapeTargetsService` | -| `attribute_mappings` ✅ | CRUD | `IngestAttributeMappingService` | -| `integrations/chat_*` ✅ | connector list (manifest + linked workspaces) + admin-only install/settings/unlink; the chat platform is a value, never a path segment | `ChatWorkspaceService` | -| `integrations/planetscale` ✅ | status + connect/organizations/`select_organization`/`metrics_token`/disconnect + databases/`webhook_config`/`query_insights`/`events` | `PlanetScaleConnectionService`, `PlanetScaleOAuthService`, `PlanetScaleService` | -| `integrations/gcp` ✅ | status (connectors + `metrics_available`) + admin-only connector create/delete and `setup_scripts` | `GcpConnectorService` | -| `session_replays` ✅ | `search`/retrieve + events/transcript/`for_trace` (reduced; `facets`/`trace-summaries` deferred) | `sessionReplays` | -| `mobile_devices` ✅ | list (mine) / `PUT …/{token}` register-or-refresh / `DELETE …/{token}`; per user, per org; drives push fan-out | `MobileDevicesService` | -| `agent_feedback` ✅ | create/list; feedback about Maple from agents: `kind`, `agent` (type/name/model/version), `reason`, optional `impact`/`details`/`related_to` | `AgentFeedbackService` | -| `organization` 🟡 | retrieve (GET only shipped); update settings (incl. ClickHouse BYOC) + delete deferred | `organizations`, `orgClickHouseSettings` | -| `traces` ✅ | search/timeseries/breakdown + direct trace/span reads | `queryEngine`, `observability` | -| `logs` ✅ | search/timeseries/breakdown + direct log reads | `queryEngine` | -| `metrics` ✅ | catalog + timeseries/breakdown | `queryEngine` | -| `services` ✅ | `GET /v2/services`, `GET /v2/services/{name}`, `GET /v2/services/{name}/overview` (summary + all-signal series + operations, one round-trip) | `queryEngine` | -| `service_map` ✅ | `GET /v2/service_map` | `queryEngine` | +| Resource | Endpoints | Backing service | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| `api_keys` ✅ pilot | list/create/retrieve/roll/revoke, `scopes` param | `ApiKeysService` | +| `ingest_keys` ✅ | retrieve, `POST …/public/roll`, `POST …/private/roll` | `OrgIngestKeysService` | +| `dashboards` ✅ | CRUD + `versions` (list/retrieve/restore) + `templates` (list/preview/instantiate) + Perses import | `dashboards` | +| `alerts/rules` ✅ | CRUD + `test` + `preview` + `checks` | `AlertsService` | +| `alerts/destinations` ✅ | CRUD + `test` | `AlertsService` | +| `alerts/incidents` ✅ | list/retrieve | `AlertsService` | +| `alerts/deliveries` ✅ | list delivery attempts | `AlertsService` | +| `error_issues` 🟡 | list/retrieve/`service_counts` ✅; `events`, `comments`, `transitions`, `assignee`, `severity` deferred | `errors` | +| `investigations` ✅ | list/retrieve/create/status | `InvestigationService` | +| `anomalies` ✅ | incidents list/retrieve/timeseries/resolve/link-issue, `service_counts` aggregate + `PATCH` settings | `anomalies` | +| `instrumentation/recommendations` ✅ | list + dismiss/reopen | `RecommendationIssueService` | +| `instrumentation/audit` ✅ | retrieve (singleton report, recomputed per request) | `SetupAuditService` | +| `onboarding/checklist` ✅ | retrieve + claim (singleton; steps verified per request, claim admin-only and idempotent) | `OnboardingChecklistService` | +| `scrape_targets` ✅ | CRUD + `probe` + `checks` | `ScrapeTargetsService` | +| `attribute_mappings` ✅ | CRUD | `IngestAttributeMappingService` | +| `integrations/chat_*` ✅ | connector list (manifest + linked workspaces) + admin-only install/settings/unlink; the chat platform is a value, never a path segment | `ChatWorkspaceService` | +| `integrations/planetscale` ✅ | status + connect/organizations/`select_organization`/`metrics_token`/disconnect + databases/`webhook_config`/`query_insights`/`events` | `PlanetScaleConnectionService`, `PlanetScaleOAuthService`, `PlanetScaleService` | +| `integrations/gcp` ✅ | status (connectors + `metrics_available`) + admin-only connector create/update/delete (project, folder or organization scope) and `setup_scripts` | `GcpConnectorService` | +| `session_replays` ✅ | `search`/retrieve + events/transcript/`for_trace` (reduced; `facets`/`trace-summaries` deferred) | `sessionReplays` | +| `mobile_devices` ✅ | list (mine) / `PUT …/{token}` register-or-refresh / `DELETE …/{token}`; per user, per org; drives push fan-out | `MobileDevicesService` | +| `agent_feedback` ✅ | create/list; feedback about Maple from agents: `kind`, `agent` (type/name/model/version), `reason`, optional `impact`/`details`/`related_to` | `AgentFeedbackService` | +| `organization` 🟡 | retrieve (GET only shipped); update settings (incl. ClickHouse BYOC) + delete deferred | `organizations`, `orgClickHouseSettings` | +| `traces` ✅ | search/timeseries/breakdown + direct trace/span reads | `queryEngine`, `observability` | +| `logs` ✅ | search/timeseries/breakdown + direct log reads | `queryEngine` | +| `metrics` ✅ | catalog + timeseries/breakdown | `queryEngine` | +| `services` ✅ | `GET /v2/services`, `GET /v2/services/{name}`, `GET /v2/services/{name}/overview` (summary + all-signal series + operations, one round-trip) | `queryEngine` | +| `service_map` ✅ | `GET /v2/service_map` | `queryEngine` | Also mounted, outside this table: `audit_log` (list), `environments`, `instrumentation/signals` (retrieve), `widget_credentials` (mint/revoke), `widget_summary` (retrieve), and `share` (share-link reads). The contracts are in `packages/domain/src/http/v2/`. diff --git a/packages/backend/src/services/audit/audit-actions.ts b/packages/backend/src/services/audit/audit-actions.ts index dcde1efaf7..8eeb093ef7 100644 --- a/packages/backend/src/services/audit/audit-actions.ts +++ b/packages/backend/src/services/audit/audit-actions.ts @@ -61,8 +61,8 @@ export const AuditResources = { dashboard_share: { prefix: PublicIdPrefixes.dashboardShare, verbs: ["created", "rotated", "deleted"] }, /** Verbs mirror the issue event types — `recordEvent` audits every one it attributes. */ error_issue: { prefix: PublicIdPrefixes.errorIssue, verbs: ErrorIssueEventType.literals }, - /** One connected Google Cloud project; the project id is in `metadata`. */ - gcp_connector: { prefix: PublicIdPrefixes.gcpConnector, verbs: ["created", "deleted"] }, + /** One connected Google Cloud scope; the scope and what it collects are in `metadata`. */ + gcp_connector: { prefix: PublicIdPrefixes.gcpConnector, verbs: ["created", "updated", "deleted"] }, /** Org-singleton public/private pair; which one rolled is in `metadata`. */ ingest_key: { verbs: ["rolled"] }, /** diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts index c7773f0623..9223f38e84 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.test.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -2,7 +2,7 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { ConfigProvider, Effect, Layer, Schema } from "effect" import { hashIngestKey } from "@maple/db" import { gcpConnectorResourceNames } from "@maple/domain/gcp" -import { GcpProjectId, OrgId, UserId } from "@maple/domain/primitives" +import { GcpProjectId, GcpResourceNumber, OrgId, UserId } from "@maple/domain/primitives" import { Env } from "@maple/backend/platform/Env" import { cleanupTestDbs, @@ -11,7 +11,7 @@ import { queryFirstRow, type TestDb, } from "@maple/backend/platform/test-pglite" -import { GcpConnectorService, type GcpConnector } from "./GcpConnectorService" +import { GcpConnectorService, type CreateGcpConnectorInput } from "./GcpConnectorService" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) @@ -48,7 +48,27 @@ const makeLayer = (testDb: TestDb, mapleServiceAccount?: string) => const orgId = Schema.decodeUnknownSync(OrgId)("org_gcp") const otherOrgId = Schema.decodeUnknownSync(OrgId)("org_other") const userId = Schema.decodeUnknownSync(UserId)("user_1") -const projectId = Schema.decodeUnknownSync(GcpProjectId) +const project = Schema.decodeUnknownSync(GcpProjectId) +const number = Schema.decodeUnknownSync(GcpResourceNumber) + +const projectScope = (id: string, flags?: Partial): CreateGcpConnectorInput => ({ + scopeType: "project", + scopeId: project(id), + projectId: project(id), + logsEnabled: true, + metricsEnabled: false, + ...flags, +}) + +const organizationScope: CreateGcpConnectorInput = { + scopeType: "organization", + scopeId: number("123456789012"), + projectId: project("acme-host"), + logsEnabled: true, + metricsEnabled: true, +} + +const scriptOptions = { excludeGkeContainerLogs: false } interface StoredConnector { readonly secret_ciphertext: string @@ -61,14 +81,13 @@ describe("GcpConnectorService", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { const gcp = yield* GcpConnectorService - const connector = yield* gcp.create(orgId, userId, projectId("acme-prod")) + const connector = yield* gcp.create(orgId, userId, projectScope("acme-prod")) + assert.strictEqual(connector.scopeId, "acme-prod") assert.strictEqual(connector.projectId, "acme-prod") assert.isNull(connector.lastLogReceivedAt) assert.isNull(connector.lastLogError) - const { setupScript } = yield* gcp.scripts(orgId, connector.id, { - excludeGkeContainerLogs: false, - }) + const { setupScript } = yield* gcp.scripts(orgId, connector.id, scriptOptions) const endpoint = /PUSH_ENDPOINT='([^']+)'/.exec(setupScript)?.[1] assert.isDefined(endpoint) const url = new URL(endpoint!) @@ -90,25 +109,82 @@ describe("GcpConnectorService", () => { }).pipe(Effect.provide(makeLayer(testDb))) }) - it.effect("refuses a project the organization already connected, but not another organization's", () => { + it.effect("connects a scope once per organization, whatever its host project", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { const gcp = yield* GcpConnectorService - yield* gcp.create(orgId, userId, projectId("acme-prod")) - const duplicate = yield* Effect.flip(gcp.create(orgId, userId, projectId("acme-prod"))) - assert.strictEqual(duplicate._tag, "@maple/http/errors/GcpProjectAlreadyConnectedError") + yield* gcp.create(orgId, userId, organizationScope) + const duplicate = yield* Effect.flip( + gcp.create(orgId, userId, { ...organizationScope, projectId: project("another-host") }), + ) + assert.strictEqual(duplicate._tag, "@maple/http/errors/GcpScopeAlreadyConnectedError") + + // A folder with the same number is a different scope, and so is a project inside it. + yield* gcp.create(orgId, userId, { ...organizationScope, scopeType: "folder" }) + yield* gcp.create(orgId, userId, projectScope("acme-host")) + yield* gcp.create(otherOrgId, userId, organizationScope) + assert.lengthOf((yield* gcp.status(orgId)).connectors, 3) + }).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))) + }) - yield* gcp.create(otherOrgId, userId, projectId("acme-prod")) - assert.lengthOf((yield* gcp.status(orgId)).connectors, 1) + it.effect("requires one capability to stay on and refuses metrics without a Google identity", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + assert.isFalse((yield* gcp.status(orgId)).metricsAvailable) + + const nothing = yield* Effect.flip( + gcp.create(orgId, userId, projectScope("acme-prod", { logsEnabled: false })), + ) + assert.strictEqual(nothing._tag, "@maple/http/errors/IntegrationsValidationError") + const metrics = yield* Effect.flip( + gcp.create(orgId, userId, projectScope("acme-prod", { metricsEnabled: true })), + ) + assert.strictEqual(metrics._tag, "@maple/http/errors/GcpMetricsUnavailableError") + + const connector = yield* gcp.create(orgId, userId, projectScope("acme-prod")) + const turnOn = yield* Effect.flip(gcp.update(orgId, connector.id, { metricsEnabled: true })) + assert.strictEqual(turnOn._tag, "@maple/http/errors/GcpMetricsUnavailableError") + const turnOff = yield* Effect.flip(gcp.update(orgId, connector.id, { logsEnabled: false })) + assert.strictEqual(turnOff._tag, "@maple/http/errors/IntegrationsValidationError") + // An empty patch changes nothing. + assert.deepStrictEqual(yield* gcp.update(orgId, connector.id, {}), connector) }).pipe(Effect.provide(makeLayer(testDb))) }) - it.effect("reports the gateway's last push and renders metrics steps only with a Google identity", () => { + it.effect("switches capabilities, and the scripts follow the stored flags", () => { const testDb = createTestDb(trackedDbs) - const withoutMetrics = Effect.gen(function* () { + return Effect.gen(function* () { const gcp = yield* GcpConnectorService - const connector = yield* gcp.create(orgId, userId, projectId("acme-prod")) - // What the ingest gateway writes after a rejected push. + assert.isTrue((yield* gcp.status(orgId)).metricsAvailable) + const connector = yield* gcp.create(orgId, userId, organizationScope) + const both = yield* gcp.scripts(orgId, connector.id, { excludeGkeContainerLogs: true }) + assert.include(both.setupScript, `MAPLE_SERVICE_ACCOUNT='${MAPLE_ACCOUNT}'`) + assert.include(both.setupScript, 'NOT resource.type="k8s_container"') + assert.include(both.setupScript, '--organization="$SCOPE_ID" --include-children') + assert.include(both.setupScript, "PROJECT_ID='acme-host'") + assert.notInclude(both.cleanupScript, "maple_gcp_") + + const metricsOnly = yield* gcp.update(orgId, connector.id, { logsEnabled: false }) + assert.deepStrictEqual(metricsOnly, { ...connector, logsEnabled: false }) + const afterOptOut = yield* gcp.scripts(orgId, connector.id, scriptOptions) + assert.notInclude(afterOptOut.setupScript, "maple_gcp_") + assert.include(afterOptOut.setupScript, "gcloud logging sinks delete") + + const logsOnly = yield* gcp.update(orgId, connector.id, { + logsEnabled: true, + metricsEnabled: false, + }) + assert.deepStrictEqual(logsOnly, { ...connector, metricsEnabled: false }) + assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, [logsOnly]) + }).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))) + }) + + it.effect("reports what the ingest gateway recorded for the last push", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + const connector = yield* gcp.create(orgId, userId, projectScope("acme-prod")) yield* Effect.promise(() => executeSql( testDb, @@ -116,42 +192,22 @@ describe("GcpConnectorService", () => { ["2026-10-08T09:12:00.000Z", "payload was not a LogEntry", connector.id], ), ) - const status = yield* gcp.status(orgId) - assert.isFalse(status.metricsAvailable) - assert.deepStrictEqual(status.connectors, [ + assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, [ { ...connector, lastLogReceivedAt: Date.parse("2026-10-08T09:12:00.000Z"), lastLogError: "payload was not a LogEntry", }, ]) - const scripts = yield* gcp.scripts(orgId, connector.id, { excludeGkeContainerLogs: true }) - assert.notInclude(scripts.setupScript, "service-accounts") - assert.include(scripts.setupScript, 'NOT resource.type="k8s_container"') - return connector }).pipe(Effect.provide(makeLayer(testDb))) - - const withMetrics = Effect.fnUntraced(function* (connectorId: GcpConnector["id"]) { - const gcp = yield* GcpConnectorService - assert.isTrue((yield* gcp.status(orgId)).metricsAvailable) - const scripts = yield* gcp.scripts(orgId, connectorId, { excludeGkeContainerLogs: false }) - assert.include(scripts.setupScript, `MAPLE_SERVICE_ACCOUNT='${MAPLE_ACCOUNT}'`) - assert.notInclude(scripts.cleanupScript, "maple_gcp_") - }) - - return withoutMetrics.pipe( - Effect.flatMap((connector) => - withMetrics(connector.id).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))), - ), - ) }) it.effect("refuses to render a secret that was moved onto another connector's row", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { const gcp = yield* GcpConnectorService - const source = yield* gcp.create(orgId, userId, projectId("acme-prod")) - const target = yield* gcp.create(orgId, userId, projectId("acme-staging")) + const source = yield* gcp.create(orgId, userId, projectScope("acme-prod")) + const target = yield* gcp.create(orgId, userId, projectScope("acme-staging")) yield* Effect.promise(() => executeSql( testDb, @@ -161,27 +217,28 @@ describe("GcpConnectorService", () => { [source.id, target.id], ), ) - const error = yield* Effect.flip( - gcp.scripts(orgId, target.id, { excludeGkeContainerLogs: false }), - ) + const error = yield* Effect.flip(gcp.scripts(orgId, target.id, scriptOptions)) assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsPersistenceError") }).pipe(Effect.provide(makeLayer(testDb))) }) - it.effect("scopes scripts and deletion to the owning organization", () => { + it.effect("scopes updates, scripts and deletion to the owning organization", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { const gcp = yield* GcpConnectorService - const connector = yield* gcp.create(orgId, userId, projectId("acme-prod")) + const connector = yield* gcp.create(orgId, userId, projectScope("acme-prod")) - const options = { excludeGkeContainerLogs: false } - const foreignScripts = yield* Effect.flip(gcp.scripts(otherOrgId, connector.id, options)) - assert.strictEqual(foreignScripts._tag, "@maple/http/errors/IntegrationsNotFoundError") - const foreignDelete = yield* Effect.flip(gcp.delete(otherOrgId, connector.id)) - assert.strictEqual(foreignDelete._tag, "@maple/http/errors/IntegrationsNotFoundError") + for (const foreign of [ + gcp.update(otherOrgId, connector.id, { logsEnabled: true }), + gcp.scripts(otherOrgId, connector.id, scriptOptions), + gcp.delete(otherOrgId, connector.id), + ]) { + const error = yield* Effect.flip(foreign) + assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsNotFoundError") + } const deleted = yield* gcp.delete(orgId, connector.id) - assert.strictEqual(deleted.projectId, "acme-prod") + assert.deepStrictEqual(deleted.connector, connector) assert.include(deleted.cleanupScript, gcpConnectorResourceNames(connector.id).sink) assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, []) const again = yield* Effect.flip(gcp.delete(orgId, connector.id)) diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index 77bf711fc6..57bf395de4 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -1,11 +1,20 @@ import { randomBytes, randomUUID } from "node:crypto" import { gcpConnectors, hashIngestKey, parseIngestKeyLookupHmacKey, type GcpConnectorRow } from "@maple/db" import { - GcpProjectAlreadyConnectedError, + GcpMetricsUnavailableError, + GcpScopeAlreadyConnectedError, IntegrationsNotFoundError, IntegrationsPersistenceError, + IntegrationsValidationError, } from "@maple/domain/http" -import { GcpConnectorId, type GcpProjectId, type OrgId, type UserId } from "@maple/domain/primitives" +import { + GcpConnectorId, + type GcpProjectId, + type GcpResourceNumber, + type GcpScopeType, + type OrgId, + type UserId, +} from "@maple/domain/primitives" import { and, asc, eq } from "drizzle-orm" import { Clock, Context, Effect, Layer, Option, Redacted } from "effect" import { decryptAes256Gcm, encryptAes256Gcm, parseBase64Aes256GcmKey } from "@maple/backend/platform/Crypto" @@ -17,17 +26,31 @@ import { renderGcpCleanupScript, renderGcpSetupScript } from "./gcp/setup-script export interface GcpConnector { readonly id: GcpConnectorId + readonly scopeType: GcpScopeType + /** The project id, or the numeric folder / organization id. */ + readonly scopeId: GcpProjectId | GcpResourceNumber + /** The host project, where Maple's own resources live. Equals `scopeId` for a project scope. */ readonly projectId: GcpProjectId + readonly logsEnabled: boolean + /** Covers metrics and resource collection. */ + readonly metricsEnabled: boolean readonly createdAt: number - /** Last log push the ingest gateway accepted from this project. Null until the first one. */ + /** Last log push the ingest gateway accepted from this connector. Null until the first one. */ readonly lastLogReceivedAt: number | null readonly lastLogError: string | null } +export type CreateGcpConnectorInput = Pick< + GcpConnector, + "scopeType" | "scopeId" | "projectId" | "logsEnabled" | "metricsEnabled" +> + +type GcpCapabilityError = GcpMetricsUnavailableError | IntegrationsValidationError + export interface GcpConnectorServiceApi { readonly status: (orgId: OrgId) => Effect.Effect< { - /** False when this deployment has no Google identity, so connectors are logs-only. */ + /** False when this deployment has no Google identity to read metrics with. */ readonly metricsAvailable: boolean readonly connectors: ReadonlyArray }, @@ -36,8 +59,20 @@ export interface GcpConnectorServiceApi { readonly create: ( orgId: OrgId, userId: UserId, - projectId: GcpProjectId, - ) => Effect.Effect + input: CreateGcpConnectorInput, + ) => Effect.Effect< + GcpConnector, + GcpScopeAlreadyConnectedError | GcpCapabilityError | IntegrationsPersistenceError + > + /** Omitted flags are unchanged. Google Cloud only follows once the setup script is re-run. */ + readonly update: ( + orgId: OrgId, + connectorId: GcpConnectorId, + patch: { readonly logsEnabled?: boolean | undefined; readonly metricsEnabled?: boolean | undefined }, + ) => Effect.Effect< + GcpConnector, + IntegrationsNotFoundError | GcpCapabilityError | IntegrationsPersistenceError + > readonly scripts: ( orgId: OrgId, connectorId: GcpConnectorId, @@ -46,12 +81,12 @@ export interface GcpConnectorServiceApi { { readonly setupScript: string; readonly cleanupScript: string }, IntegrationsNotFoundError | IntegrationsPersistenceError > - /** Deleting the row is the disconnect: the ingest gateway stops accepting the project's pushes. */ + /** Deleting the row is the disconnect: the ingest gateway stops accepting the connector's pushes. */ readonly delete: ( orgId: OrgId, connectorId: GcpConnectorId, ) => Effect.Effect< - { readonly projectId: GcpProjectId; readonly cleanupScript: string }, + { readonly connector: GcpConnector; readonly cleanupScript: string }, IntegrationsNotFoundError | IntegrationsPersistenceError > } @@ -66,7 +101,11 @@ const secretAad = (connectorId: string) => Buffer.from(`gcp_connectors:v1:${conn const toConnector = (row: GcpConnectorRow): GcpConnector => ({ id: row.id, + scopeType: row.scopeType, + scopeId: row.scopeId, projectId: row.projectId, + logsEnabled: row.logsEnabled, + metricsEnabled: row.metricsEnabled, createdAt: dateToMs(row.createdAt), lastLogReceivedAt: dateToMs(row.lastReceivedAt), lastLogError: row.lastError, @@ -96,6 +135,42 @@ export class GcpConnectorService extends Context.Service => { + if (!flags.logsEnabled && !flags.metricsEnabled) { + return Effect.fail( + new IntegrationsValidationError({ + message: + "At least one of logs_enabled and metrics_enabled must be on. Delete the connector to disconnect.", + }), + ) + } + if (turningMetricsOn && mapleServiceAccountEmail === undefined) { + return Effect.fail( + new GcpMetricsUnavailableError({ + message: + "Metrics and resource collection is not available on this Maple deployment.", + }), + ) + } + return Effect.void + } + + const selectRow = (orgId: OrgId, connectorId: GcpConnectorId) => + dbExecute((db) => + db + .select() + .from(gcpConnectors) + .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) + .limit(1), + ).pipe( + Effect.flatMap(([row]) => + row === undefined ? Effect.fail(notFound()) : Effect.succeed(row), + ), + ) + const status = Effect.fn("GcpConnectorService.status")(function* (orgId: OrgId) { yield* Effect.annotateCurrentSpan({ orgId }) const rows = yield* dbExecute((db) => @@ -114,9 +189,14 @@ export class GcpConnectorService extends Context.Service + const [row] = yield* dbExecute((db) => db .insert(gcpConnectors) .values({ id, orgId, - projectId, + ...input, secretCiphertext: encrypted.ciphertext, secretIv: encrypted.iv, secretTag: encrypted.tag, @@ -141,34 +221,55 @@ export class GcpConnectorService extends Context.Service + const current = yield* selectRow(orgId, connectorId) + const flags = { + logsEnabled: patch.logsEnabled ?? current.logsEnabled, + metricsEnabled: patch.metricsEnabled ?? current.metricsEnabled, + } + yield* checkCapabilities(flags, patch.metricsEnabled === true) + const updatedAt = msToDate(yield* Clock.currentTimeMillis) + const [row] = yield* dbExecute((db) => db - .select() - .from(gcpConnectors) + .update(gcpConnectors) + .set({ ...flags, updatedAt }) .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) - .limit(1), + .returning(), ) - const row = rows[0] if (row === undefined) return yield* notFound() + return toConnector(row) + }) + + const scripts = Effect.fn("GcpConnectorService.scripts")(function* ( + orgId: OrgId, + connectorId: GcpConnectorId, + options: { readonly excludeGkeContainerLogs: boolean }, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.gcp.connector_id": connectorId }) + const row = yield* selectRow(orgId, connectorId) const secret = yield* decryptAes256Gcm( { ciphertext: row.secretCiphertext, iv: row.secretIv, tag: row.secretTag }, encryptionKey, @@ -178,15 +279,22 @@ export class GcpConnectorService extends Context.Service + const [row] = yield* dbExecute((db) => db .delete(gcpConnectors) .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) - .returning({ projectId: gcpConnectors.projectId }), + .returning(), ) - const row = rows[0] if (row === undefined) return yield* notFound() return { - projectId: row.projectId, - cleanupScript: renderGcpCleanupScript(connectorId, row.projectId), + connector: toConnector(row), + cleanupScript: renderGcpCleanupScript({ + connectorId, + scopeType: row.scopeType, + scopeId: row.scopeId, + projectId: row.projectId, + }), } }) - return { status, create, scripts, delete: deleteConnector } satisfies GcpConnectorServiceApi + return { + status, + create, + update, + scripts, + delete: deleteConnector, + } satisfies GcpConnectorServiceApi }), }, ) { diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index dd0f7b9463..d8f750dce0 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -4,28 +4,63 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { afterEach, describe, expect, it } from "vitest" import { Schema } from "effect" -import { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" +import { GcpConnectorId, GcpProjectId, GcpResourceNumber } from "@maple/domain/primitives" import { gcpLogFilter, renderGcpCleanupScript, renderGcpSetupScript, + type GcpScriptTarget, type GcpSetupScriptInput, } from "./setup-scripts" const connectorId = Schema.decodeUnknownSync(GcpConnectorId)("018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8") -const projectId = Schema.decodeUnknownSync(GcpProjectId)("acme-prod") +const hostProject = Schema.decodeUnknownSync(GcpProjectId)("acme-host") +const resourceNumber = Schema.decodeUnknownSync(GcpResourceNumber)("123456789012") const NAME = "maple-018f2b3c4d5e4f708192a3b4" +const ACCOUNT = `${NAME}@acme-host.iam.gserviceaccount.com` const SECRET = "maple_gcp_s3cr3t-value" const WRITER = "serviceAccount:service-1@gcp-sa-logging.iam.gserviceaccount.com" const MAPLE_ACCOUNT = "collector@maple-prod.iam.gserviceaccount.com" +const PUSH_ENDPOINT = `https://ingest.test/v1/logpush/gcp/${connectorId}?secret=${SECRET}` -const input: GcpSetupScriptInput = { - connectorId, - projectId, - pushEndpoint: `https://ingest.test/v1/logpush/gcp/${connectorId}?secret=${SECRET}`, +/** Each scope with the sink flag and IAM command the script must use for it. */ +const SCOPES = { + project: { + target: { connectorId, scopeType: "project", scopeId: hostProject, projectId: hostProject }, + sink: "--project=acme-host", + sinkWrite: "--project=acme-host", + iam: "projects", + id: "acme-host", + }, + folder: { + target: { connectorId, scopeType: "folder", scopeId: resourceNumber, projectId: hostProject }, + sink: "--folder=123456789012", + sinkWrite: "--folder=123456789012 --include-children", + iam: "resource-manager folders", + id: "123456789012", + }, + organization: { + target: { connectorId, scopeType: "organization", scopeId: resourceNumber, projectId: hostProject }, + sink: "--organization=123456789012", + sinkWrite: "--organization=123456789012 --include-children", + iam: "organizations", + id: "123456789012", + }, +} as const satisfies Record> + +const scopeTypes = ["project", "folder", "organization"] as const + +const setupInput = ( + scope: keyof typeof SCOPES, + capabilities: { logs: boolean; metrics: boolean }, +): GcpSetupScriptInput => ({ + ...SCOPES[scope].target, + pushEndpoint: PUSH_ENDPOINT, mapleServiceAccountEmail: MAPLE_ACCOUNT, + logsEnabled: capabilities.logs, + metricsEnabled: capabilities.metrics, excludeGkeContainerLogs: false, -} +}) // A `gcloud` that records its arguments. `describe` reports "not found" unless the test says // the resources exist, and the sink's writer identity is the one value a script reads back. @@ -34,7 +69,7 @@ echo "$*" >> "$GCLOUD_LOG" case "$*" in *"sinks describe"*"--format"*) echo "${WRITER}" ;; *" describe "*) [ "$GCLOUD_EXISTS" = 1 ] || exit 1 ;; - *" delete "*) exit "$GCLOUD_DELETE_STATUS" ;; + *"remove-iam-policy-binding"*) exit "$GCLOUD_REMOVE_BINDING_STATUS" ;; esac ` @@ -44,7 +79,7 @@ afterEach(() => { }) /** Run a rendered script in bash against the fake gcloud; returns the commands it issued. */ -const run = (script: string, options: { exists?: boolean; deleteStatus?: number } = {}) => { +const run = (script: string, options: { exists?: boolean; removeBindingStatus?: number } = {}) => { const dir = mkdtempSync(join(tmpdir(), "maple-gcp-script-")) tempDirs.push(dir) writeFileSync(join(dir, "gcloud"), FAKE_GCLOUD) @@ -59,7 +94,7 @@ const run = (script: string, options: { exists?: boolean; deleteStatus?: number PATH: `${dir}:${process.env.PATH}`, GCLOUD_LOG: log, GCLOUD_EXISTS: options.exists === true ? "1" : "0", - GCLOUD_DELETE_STATUS: String(options.deleteStatus ?? 0), + GCLOUD_REMOVE_BINDING_STATUS: String(options.removeBindingStatus ?? 0), }, }) return { @@ -70,51 +105,121 @@ const run = (script: string, options: { exists?: boolean; deleteStatus?: number } } -describe("renderGcpSetupScript", () => { - it("creates the log pipeline and the metrics account in a fresh project", () => { - const { status, stdout, commands } = run(renderGcpSetupScript(input)) +const logsSetupCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ + "services enable pubsub.googleapis.com logging.googleapis.com --project=acme-host", + `pubsub topics describe ${NAME} --project=acme-host`, + `pubsub topics create ${NAME} --project=acme-host`, + `pubsub subscriptions describe ${NAME} --project=acme-host`, + `pubsub subscriptions create ${NAME} --topic=${NAME} --project=acme-host --push-endpoint=${PUSH_ENDPOINT} --push-no-wrapper --ack-deadline=30 --message-retention-duration=1d --expiration-period=never --min-retry-delay=10s --max-retry-delay=600s`, + `logging sinks describe ${NAME} ${scope.sink}`, + `logging sinks create ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(false)} ${scope.sinkWrite}`, + `logging sinks describe ${NAME} ${scope.sink} --format=value(writerIdentity)`, + `projects add-iam-policy-binding acme-host --member=${WRITER} --role=roles/logging.logWriter --condition=None`, + `pubsub topics add-iam-policy-binding ${NAME} --project=acme-host --member=${WRITER} --role=roles/pubsub.publisher`, +] + +const metricsSetupCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ + "services enable monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com iamcredentials.googleapis.com --project=acme-host", + `iam service-accounts describe ${ACCOUNT} --project=acme-host`, + `iam service-accounts create ${NAME} --project=acme-host --display-name=Maple metrics and resource reader`, + `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, + `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, + `projects add-iam-policy-binding acme-host --member=serviceAccount:${ACCOUNT} --role=roles/serviceusage.serviceUsageConsumer --condition=None`, + `iam service-accounts add-iam-policy-binding ${ACCOUNT} --project=acme-host --member=serviceAccount:${MAPLE_ACCOUNT} --role=roles/iam.serviceAccountTokenCreator`, +] + +const logsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ + `logging sinks describe ${NAME} ${scope.sink}`, + `logging sinks delete ${NAME} ${scope.sink} --quiet`, + `pubsub subscriptions describe ${NAME} --project=acme-host`, + `pubsub subscriptions delete ${NAME} --project=acme-host --quiet`, + `pubsub topics describe ${NAME} --project=acme-host`, + `pubsub topics delete ${NAME} --project=acme-host --quiet`, +] + +const metricsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ + `iam service-accounts describe ${ACCOUNT} --project=acme-host`, + `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, + `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, + `projects remove-iam-policy-binding acme-host --member=serviceAccount:${ACCOUNT} --role=roles/serviceusage.serviceUsageConsumer --condition=None`, + `iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`, +] + +describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { + const scope = SCOPES[scopeType] + + it("sets up logs and metrics from nothing, at the right scope", () => { + const { status, stdout, commands } = run( + renderGcpSetupScript(setupInput(scopeType, { logs: true, metrics: true })), + ) expect(status).toBe(0) - expect(stdout).toContain("Maple setup complete for acme-prod") - expect(commands).toEqual([ - "services enable pubsub.googleapis.com logging.googleapis.com monitoring.googleapis.com iam.googleapis.com iamcredentials.googleapis.com --project=acme-prod", - `pubsub topics describe ${NAME} --project=acme-prod`, - `pubsub topics create ${NAME} --project=acme-prod`, - `pubsub subscriptions describe ${NAME} --project=acme-prod`, - `pubsub subscriptions create ${NAME} --topic=${NAME} --project=acme-prod --push-endpoint=${input.pushEndpoint} --push-no-wrapper --ack-deadline=30 --message-retention-duration=1d --expiration-period=never --min-retry-delay=10s --max-retry-delay=600s`, - `logging sinks describe ${NAME} --project=acme-prod`, - `logging sinks create ${NAME} pubsub.googleapis.com/projects/acme-prod/topics/${NAME} --log-filter=${gcpLogFilter(false)} --project=acme-prod`, - `logging sinks describe ${NAME} --project=acme-prod --format=value(writerIdentity)`, - `projects add-iam-policy-binding acme-prod --member=${WRITER} --role=roles/logging.logWriter --condition=None`, - `pubsub topics add-iam-policy-binding ${NAME} --project=acme-prod --member=${WRITER} --role=roles/pubsub.publisher`, - `iam service-accounts describe ${NAME}@acme-prod.iam.gserviceaccount.com --project=acme-prod`, - `iam service-accounts create ${NAME} --project=acme-prod --display-name=Maple metrics reader`, - `projects add-iam-policy-binding acme-prod --member=serviceAccount:${NAME}@acme-prod.iam.gserviceaccount.com --role=roles/monitoring.viewer --condition=None`, - `iam service-accounts add-iam-policy-binding ${NAME}@acme-prod.iam.gserviceaccount.com --project=acme-prod --member=serviceAccount:${MAPLE_ACCOUNT} --role=roles/iam.serviceAccountTokenCreator`, - ]) + expect(stdout).toContain("Maple setup complete.") + expect(commands).toEqual([...logsSetupCommands(scope), ...metricsSetupCommands(scope)]) }) - it("updates instead of creating when re-run, so an edited filter is applied", () => { - const { status, commands } = run(renderGcpSetupScript(input), { exists: true }) + it("with logs only, sets up logs and removes an earlier metrics setup", () => { + const script = renderGcpSetupScript(setupInput(scopeType, { logs: true, metrics: false })) + expect(script).not.toContain(MAPLE_ACCOUNT) + const { status, commands } = run(script, { exists: true }) expect(status).toBe(0) + // Everything exists already, so the log pipeline is updated in place. expect(commands.filter((command) => command.includes(" create "))).toEqual([]) + expect(commands).toContain( + `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(false)} ${scope.sinkWrite}`, + ) expect(commands).toContainEqual( expect.stringMatching(/^pubsub subscriptions update .*--push-no-wrapper/), ) - expect(commands).toContainEqual( - expect.stringMatching(/^logging sinks update .*--log-filter=NOT log_id/), + expect(commands.slice(-metricsRemovalCommands(scope).length)).toEqual(metricsRemovalCommands(scope)) + }) + + it("with metrics only, removes an earlier log pipeline and carries no secret", () => { + const script = renderGcpSetupScript(setupInput(scopeType, { logs: false, metrics: true })) + expect(script).not.toContain(SECRET) + expect(script).not.toContain("LOG_FILTER") + const { status, commands } = run(script, { exists: true }) + expect(status).toBe(0) + expect(commands.slice(0, logsRemovalCommands(scope).length)).toEqual(logsRemovalCommands(scope)) + expect(commands).not.toContainEqual(expect.stringContaining("pubsub.googleapis.com logging")) + expect(commands).toContain( + `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, ) }) - it("omits every metrics step when the deployment has no Google identity", () => { - const script = renderGcpSetupScript({ ...input, mapleServiceAccountEmail: undefined }) - expect(script).not.toContain("service-accounts") - expect(script).not.toContain("monitoring") + it("cleans up everything, and touches nothing when nothing exists", () => { + const script = renderGcpCleanupScript(scope.target) + expect(script).not.toContain("secret") + const existing = run(script, { exists: true }) + expect(existing.status).toBe(0) + expect(existing.commands).toEqual([...logsRemovalCommands(scope), ...metricsRemovalCommands(scope)]) + + const fresh = run(script) + expect(fresh.status).toBe(0) + expect(fresh.commands.every((command) => command.includes(" describe "))).toBe(true) + }) +}) + +describe("renderGcpSetupScript", () => { + it("treats metrics as off when the deployment has no Google identity", () => { + const script = renderGcpSetupScript({ + ...setupInput("organization", { logs: true, metrics: true }), + mapleServiceAccountEmail: undefined, + }) const { status, commands } = run(script) expect(status).toBe(0) - expect(commands[0]).toBe( - "services enable pubsub.googleapis.com logging.googleapis.com --project=acme-prod", - ) - expect(commands.at(-1)).toContain("pubsub topics add-iam-policy-binding") + expect(commands).toEqual([ + ...logsSetupCommands(SCOPES.organization), + `iam service-accounts describe ${ACCOUNT} --project=acme-host`, + ]) + }) + + it("still deletes the service account when one of its role bindings is already gone", () => { + const { status, commands } = run(renderGcpCleanupScript(SCOPES.folder.target), { + exists: true, + removeBindingStatus: 1, + }) + expect(status).toBe(0) + expect(commands.at(-1)).toBe(`iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`) }) it("excludes data-access audit logs and health-check probes by default, GKE containers on request", () => { @@ -122,40 +227,24 @@ describe("renderGcpSetupScript", () => { 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC"', ) expect(gcpLogFilter(true)).toBe(`${gcpLogFilter(false)} AND NOT resource.type="k8s_container"`) - expect(renderGcpSetupScript({ ...input, excludeGkeContainerLogs: true })).toContain( - `LOG_FILTER='${gcpLogFilter(true)}'`, - ) + expect( + renderGcpSetupScript({ + ...setupInput("project", { logs: true, metrics: false }), + excludeGkeContainerLogs: true, + }), + ).toContain(`LOG_FILTER='${gcpLogFilter(true)}'`) }) it("passes interpolated values to gcloud verbatim, whatever they contain", () => { const hostile = `x'; touch pwned; echo '$(touch pwned)` const { dir, status, commands } = run( - renderGcpSetupScript({ ...input, mapleServiceAccountEmail: hostile }), + renderGcpSetupScript({ + ...setupInput("project", { logs: false, metrics: true }), + mapleServiceAccountEmail: hostile, + }), ) expect(status).toBe(0) expect(commands.at(-1)).toContain(`--member=serviceAccount:${hostile} `) expect(existsSync(join(dir, "pwned"))).toBe(false) }) }) - -describe("renderGcpCleanupScript", () => { - const script = renderGcpCleanupScript(connectorId, projectId) - - it("carries the secret in the setup script only", () => { - expect(renderGcpSetupScript(input)).toContain(SECRET) - expect(script).not.toContain("secret") - expect(script).not.toContain("maple_gcp_") - }) - - it("removes every resource, continuing past ones that are already gone", () => { - const { status, commands } = run(script, { deleteStatus: 1 }) - expect(status).toBe(0) - expect(commands).toEqual([ - `logging sinks delete ${NAME} --project=acme-prod --quiet`, - `pubsub subscriptions delete ${NAME} --project=acme-prod`, - `pubsub topics delete ${NAME} --project=acme-prod`, - `projects remove-iam-policy-binding acme-prod --member=serviceAccount:${NAME}@acme-prod.iam.gserviceaccount.com --role=roles/monitoring.viewer --condition=None`, - `iam service-accounts delete ${NAME}@acme-prod.iam.gserviceaccount.com --project=acme-prod --quiet`, - ]) - }) -}) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 4081551640..cb37f7e6bd 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -1,5 +1,10 @@ import { gcpConnectorResourceNames } from "@maple/domain/gcp" -import type { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" +import type { + GcpConnectorId, + GcpProjectId, + GcpResourceNumber, + GcpScopeType, +} from "@maple/domain/primitives" /** Single-quote a value for bash. Every value that reaches a script goes through here. */ const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` @@ -16,26 +21,127 @@ export const gcpLogFilter = (excludeGkeContainerLogs: boolean): string => " AND ", ) -const logVariables = (connectorId: GcpConnectorId, projectId: GcpProjectId): string => { - const names = gcpConnectorResourceNames(connectorId) - return `PROJECT_ID=${sh(projectId)} +/** What a connector covers and where Maple's own resources live. */ +export interface GcpScriptTarget { + readonly connectorId: GcpConnectorId + readonly scopeType: GcpScopeType + readonly scopeId: GcpProjectId | GcpResourceNumber + /** The host project: topic, subscription and service account are created here. */ + readonly projectId: GcpProjectId +} + +// How each scope is addressed: the flag that places a log sink, the command family that edits its +// IAM policy, and the roles the person running the script needs there. +const SCOPES = { + project: { + sink: '--project="$SCOPE_ID"', + iam: "gcloud projects", + needs: "# Owner on the project (PROJECT_ID below).", + }, + folder: { + sink: '--folder="$SCOPE_ID"', + iam: "gcloud resource-manager folders", + needs: `# Owner on the host project (PROJECT_ID below), which holds Maple's Pub/Sub topic, +# subscription and service account. On the folder (SCOPE_ID below): Logs Configuration +# Writer for the log sink, and Folder IAM Admin for the read-only roles.`, + }, + organization: { + sink: '--organization="$SCOPE_ID"', + iam: "gcloud organizations", + needs: `# Owner on the host project (PROJECT_ID below), which holds Maple's Pub/Sub topic, +# subscription and service account. On the organization (SCOPE_ID below): Logs Configuration +# Writer for the log sink, and Organization Administrator for the read-only roles.`, + }, +} as const + +// A folder or organization sink also routes the logs of every project underneath. +const includeChildren = (scopeType: GcpScopeType): string => + scopeType === "project" ? "" : " --include-children" + +const variables = (target: GcpScriptTarget): string => { + const names = gcpConnectorResourceNames(target.connectorId) + return `PROJECT_ID=${sh(target.projectId)} +SCOPE_ID=${sh(target.scopeId)} TOPIC=${sh(names.topic)} SUBSCRIPTION=${sh(names.subscription)} -SINK=${sh(names.sink)}` +SINK=${sh(names.sink)} +SERVICE_ACCOUNT=${sh(names.serviceAccountId)} +SERVICE_ACCOUNT_EMAIL="$SERVICE_ACCOUNT@$PROJECT_ID.iam.gserviceaccount.com"` } -const serviceAccountVariables = (connectorId: GcpConnectorId): string => - `SERVICE_ACCOUNT=${sh(gcpConnectorResourceNames(connectorId).serviceAccountId)} -SERVICE_ACCOUNT_EMAIL="$SERVICE_ACCOUNT@$PROJECT_ID.iam.gserviceaccount.com"` +const logsSetup = (scopeType: GcpScopeType, pushEndpoint: string): string => ` +# ---- Logs: on ---- +PUSH_ENDPOINT=${sh(pushEndpoint)} +DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" + +# APIs in the host project: Pub/Sub carries the log entries, Cloud Logging routes them. +gcloud services enable pubsub.googleapis.com logging.googleapis.com --project="$PROJECT_ID" + +if ! gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud pubsub topics create "$TOPIC" --project="$PROJECT_ID" +fi -const metricsSteps = (connectorId: GcpConnectorId, mapleServiceAccountEmail: string): string => ` -# Metrics: a read-only service account in this project that Maple may impersonate. -${serviceAccountVariables(connectorId)} +# One LogEntry per request, unwrapped. A day of retention bounds what is replayed after an +# outage, and the subscription never expires while it is idle. +SUBSCRIPTION_FLAGS=( + --project="$PROJECT_ID" + --push-endpoint="$PUSH_ENDPOINT" --push-no-wrapper + --ack-deadline=30 --message-retention-duration=1d --expiration-period=never + --min-retry-delay=10s --max-retry-delay=600s +) +if gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud pubsub subscriptions update "$SUBSCRIPTION" "\${SUBSCRIPTION_FLAGS[@]}" +else + gcloud pubsub subscriptions create "$SUBSCRIPTION" --topic="$TOPIC" "\${SUBSCRIPTION_FLAGS[@]}" +fi + +if gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} >/dev/null 2>&1; then + gcloud logging sinks update "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} +else + gcloud logging sinks create "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} +fi + +# The sink writes as a Google-managed identity. Google requires Logs Writer for it on the project +# that holds the destination, and it needs Pub/Sub Publisher on this one topic. +WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} --format='value(writerIdentity)')" +gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ + --member="$WRITER_IDENTITY" --role=roles/logging.logWriter --condition=None >/dev/null +gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ + --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher >/dev/null +` + +const logsRemoval = (scopeType: GcpScopeType): string => ` +# ---- Logs: off. Remove what an earlier run created. ---- +# The Logs Writer grant on the host project stays: the sink wrote as the logging service agent +# of the ${scopeType}, an identity every other sink there shares. +if gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} >/dev/null 2>&1; then + gcloud logging sinks delete "$SINK" ${SCOPES[scopeType].sink} --quiet +fi +if gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" --quiet +fi +if gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID" >/dev/null 2>&1; then + gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" --quiet +fi +` + +const serviceAccountMember = '--member="serviceAccount:$SERVICE_ACCOUNT_EMAIL"' + +const metricsSetup = (scopeType: GcpScopeType, mapleServiceAccountEmail: string): string => ` +# ---- Metrics and resources: on ---- MAPLE_SERVICE_ACCOUNT=${sh(mapleServiceAccountEmail)} +# APIs in the host project, the project Maple's reads are made through: +# monitoring.googleapis.com Cloud Monitoring, to read metrics +# cloudasset.googleapis.com Cloud Asset Inventory, to list resources +# iam.googleapis.com to create the service account below +# iamcredentials.googleapis.com short-lived tokens for that account; no key is ever created +gcloud services enable monitoring.googleapis.com cloudasset.googleapis.com \\ + iam.googleapis.com iamcredentials.googleapis.com --project="$PROJECT_ID" + if ! gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" >/dev/null 2>&1; then gcloud iam service-accounts create "$SERVICE_ACCOUNT" --project="$PROJECT_ID" \\ - --display-name="Maple metrics reader" + --display-name="Maple metrics and resource reader" fi # A new service account can take a minute to become visible to IAM. @@ -47,106 +153,99 @@ retry() { "$@" } +# Read-only roles on the ${scopeType}${scopeType === "project" ? "" : ", inherited by every project under it"}: +# roles/monitoring.viewer read metrics (monitoring.timeSeries.list) +# roles/cloudasset.viewer list resources (cloudasset.assets.searchAllResources) +for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do + retry ${SCOPES[scopeType].iam} add-iam-policy-binding "$SCOPE_ID" \\ + ${serviceAccountMember} --role="$ROLE" --condition=None >/dev/null +done + +# Cloud Asset Inventory checks serviceusage.services.use on the project its API is called through. retry gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ - --member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" --role=roles/monitoring.viewer \\ - --condition=None >/dev/null + ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None >/dev/null +# Maple's own account may mint short-lived tokens for this one service account, nothing else. retry gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" \\ --member="serviceAccount:$MAPLE_SERVICE_ACCOUNT" --role=roles/iam.serviceAccountTokenCreator >/dev/null ` -export interface GcpSetupScriptInput { - readonly connectorId: GcpConnectorId - readonly projectId: GcpProjectId +const metricsRemoval = (scopeType: GcpScopeType): string => ` +# ---- Metrics and resources: off. Remove what an earlier run created. ---- +if gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" >/dev/null 2>&1; then + # Role bindings first, or the deleted account would stay listed in the IAM policy. A binding + # that is already gone is reported and skipped. + for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do + ${SCOPES[scopeType].iam} remove-iam-policy-binding "$SCOPE_ID" \\ + ${serviceAccountMember} --role="$ROLE" --condition=None >/dev/null || true + done + gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ + ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None >/dev/null || true + gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet +fi +` + +export interface GcpSetupScriptInput extends GcpScriptTarget { /** The ingest gateway's receiver URL for this connector, including its secret. */ readonly pushEndpoint: string - /** Maple's own Google service account. Undefined renders a logs-only script. */ + /** Maple's own Google service account. Undefined: metrics cannot be set up. */ readonly mapleServiceAccountEmail: string | undefined + readonly logsEnabled: boolean + readonly metricsEnabled: boolean readonly excludeGkeContainerLogs: boolean } /** - * The script a project owner runs in Cloud Shell: a Log Router sink into a Pub/Sub topic whose - * push subscription delivers each LogEntry to Maple. Every step first looks for the resource, so - * re-running applies an edited filter instead of failing on what already exists. + * The script an administrator runs in Cloud Shell. Logs: a Log Router sink on the scope into a + * Pub/Sub topic whose push subscription delivers each LogEntry to Maple. Metrics and resources: a + * read-only service account Maple may impersonate. The script converges: what the connector has + * enabled is created or updated, what it has not is removed, so a re-run applies an opt-out. */ export const renderGcpSetupScript = (input: GcpSetupScriptInput): string => { - const mapleAccount = input.mapleServiceAccountEmail - const metrics = mapleAccount !== undefined - return `#!/usr/bin/env bash -# Maple: forward this Google Cloud project's logs${metrics ? " and metrics" : ""} to Maple. -# Run in Cloud Shell as a project owner. Safe to re-run. -# Keep this script private: PUSH_ENDPOINT contains this connector's secret. -set -euo pipefail - + const mapleAccount = input.metricsEnabled ? input.mapleServiceAccountEmail : undefined + const logs = input.logsEnabled + ? { + note: "#\n# Keep this script private: PUSH_ENDPOINT contains this connector's secret.\n", + filter: ` # ---- Edit this to change which logs are forwarded, then re-run. ---- # Cloud Logging query language: https://cloud.google.com/logging/docs/view/logging-query-language LOG_FILTER=${sh(gcpLogFilter(input.excludeGkeContainerLogs))} # -------------------------------------------------------------------- - -${logVariables(input.connectorId, input.projectId)} -PUSH_ENDPOINT=${sh(input.pushEndpoint)} -DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" - -gcloud services enable pubsub.googleapis.com logging.googleapis.com${ - metrics ? " monitoring.googleapis.com iam.googleapis.com iamcredentials.googleapis.com" : "" - } \\ - --project="$PROJECT_ID" - -if ! gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID" >/dev/null 2>&1; then - gcloud pubsub topics create "$TOPIC" --project="$PROJECT_ID" -fi - -# One LogEntry per request, unwrapped. A day of retention bounds what is replayed after an -# outage, and the subscription never expires while it is idle. -SUBSCRIPTION_FLAGS=( - --project="$PROJECT_ID" - --push-endpoint="$PUSH_ENDPOINT" --push-no-wrapper - --ack-deadline=30 --message-retention-duration=1d --expiration-period=never - --min-retry-delay=10s --max-retry-delay=600s -) -if gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID" >/dev/null 2>&1; then - gcloud pubsub subscriptions update "$SUBSCRIPTION" "\${SUBSCRIPTION_FLAGS[@]}" -else - gcloud pubsub subscriptions create "$SUBSCRIPTION" --topic="$TOPIC" "\${SUBSCRIPTION_FLAGS[@]}" -fi - -if gcloud logging sinks describe "$SINK" --project="$PROJECT_ID" >/dev/null 2>&1; then - gcloud logging sinks update "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" --project="$PROJECT_ID" -else - gcloud logging sinks create "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" --project="$PROJECT_ID" -fi - -# The sink writes as a Google-managed identity, which needs to route logs and publish to the topic. -WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" --project="$PROJECT_ID" --format='value(writerIdentity)')" -gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ - --member="$WRITER_IDENTITY" --role=roles/logging.logWriter --condition=None >/dev/null -gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ - --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher >/dev/null -${mapleAccount === undefined ? "" : metricsSteps(input.connectorId, mapleAccount)} -echo "Maple setup complete for $PROJECT_ID. Logs start arriving in Maple within a few minutes." +`, + section: logsSetup(input.scopeType, input.pushEndpoint), + } + : { note: "", filter: "", section: logsRemoval(input.scopeType) } + return `#!/usr/bin/env bash +# Maple: connect a Google Cloud ${input.scopeType} to Maple. +# Logs: ${input.logsEnabled ? "on" : "off"} +# Metrics and resources: ${mapleAccount === undefined ? "off" : "on"} +# Run in Cloud Shell. Safe to re-run: it sets up what is on and removes what is off. +# +# You need: +${SCOPES[input.scopeType].needs} +${logs.note}set -euo pipefail +${logs.filter} +${variables(input)} +${logs.section}${ + mapleAccount === undefined + ? metricsRemoval(input.scopeType) + : metricsSetup(input.scopeType, mapleAccount) + } +echo "Maple setup complete." ` } /** Removes what the setup script created. Carries no secret, so it outlives the connector. */ -export const renderGcpCleanupScript = (connectorId: GcpConnectorId, projectId: GcpProjectId): string => +export const renderGcpCleanupScript = (target: GcpScriptTarget): string => `#!/usr/bin/env bash -# Maple: remove what the Maple setup script created in this Google Cloud project. -# Run in Cloud Shell as a project owner. No "set -e": every step runs, and a -# NOT_FOUND error only means that resource is already gone. The logging service -# agent keeps its Logs Writer role: every sink in the project shares that identity. -set -uo pipefail - -${logVariables(connectorId, projectId)} -${serviceAccountVariables(connectorId)} - -gcloud logging sinks delete "$SINK" --project="$PROJECT_ID" --quiet -gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" -gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" -gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ - --member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" --role=roles/monitoring.viewer \\ - --condition=None >/dev/null -gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet - -echo "Maple cleanup finished for $PROJECT_ID." +# Maple: remove what the Maple setup script created for a Google Cloud ${target.scopeType}. +# Run in Cloud Shell. Safe to re-run. APIs the setup enabled stay enabled. +# +# You need: +${SCOPES[target.scopeType].needs} +set -euo pipefail + +${variables(target)} +${logsRemoval(target.scopeType)}${metricsRemoval(target.scopeType)} +echo "Maple cleanup complete." ` diff --git a/packages/db/drizzle/20261008152420_gcp_connectors/migration.sql b/packages/db/drizzle/20261008171958_gcp_connectors/migration.sql similarity index 65% rename from packages/db/drizzle/20261008152420_gcp_connectors/migration.sql rename to packages/db/drizzle/20261008171958_gcp_connectors/migration.sql index 56d4e5a5a0..73d9a68a2a 100644 --- a/packages/db/drizzle/20261008152420_gcp_connectors/migration.sql +++ b/packages/db/drizzle/20261008171958_gcp_connectors/migration.sql @@ -1,7 +1,11 @@ CREATE TABLE "gcp_connectors" ( "id" text PRIMARY KEY, "org_id" text NOT NULL, + "scope_type" text NOT NULL, + "scope_id" text NOT NULL, "project_id" text NOT NULL, + "logs_enabled" boolean DEFAULT true NOT NULL, + "metrics_enabled" boolean DEFAULT false NOT NULL, "secret_ciphertext" text NOT NULL, "secret_iv" text NOT NULL, "secret_tag" text NOT NULL, @@ -13,5 +17,5 @@ CREATE TABLE "gcp_connectors" ( "updated_at" timestamp with time zone NOT NULL ); --> statement-breakpoint -CREATE UNIQUE INDEX "gcp_connectors_org_project_idx" ON "gcp_connectors" ("org_id","project_id");--> statement-breakpoint +CREATE UNIQUE INDEX "gcp_connectors_org_scope_idx" ON "gcp_connectors" ("org_id","scope_type","scope_id");--> statement-breakpoint CREATE UNIQUE INDEX "gcp_connectors_secret_hash_unique" ON "gcp_connectors" ("secret_hash"); \ No newline at end of file diff --git a/packages/db/drizzle/20261008152420_gcp_connectors/snapshot.json b/packages/db/drizzle/20261008171958_gcp_connectors/snapshot.json similarity index 99% rename from packages/db/drizzle/20261008152420_gcp_connectors/snapshot.json rename to packages/db/drizzle/20261008171958_gcp_connectors/snapshot.json index 7486106312..e1e30a3018 100644 --- a/packages/db/drizzle/20261008152420_gcp_connectors/snapshot.json +++ b/packages/db/drizzle/20261008171958_gcp_connectors/snapshot.json @@ -1,7 +1,7 @@ { "version": "8", "dialect": "postgres", - "id": "cede4985-423f-4e80-bdb7-25d4319189ed", + "id": "c2de92d1-c56e-4b42-b7af-16fa811a694e", "prevIds": [ "a9ce02f8-139d-4eb9-bf09-315c5936b0fa" ], @@ -4117,6 +4117,32 @@ "schema": "public", "table": "gcp_connectors" }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scope_type", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scope_id", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, { "type": "text", "typeSchema": null, @@ -4130,6 +4156,32 @@ "schema": "public", "table": "gcp_connectors" }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "logs_enabled", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "metrics_enabled", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, { "type": "text", "typeSchema": null, @@ -14633,7 +14685,14 @@ "opclass": null }, { - "value": "project_id", + "value": "scope_type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "scope_id", "isExpression": false, "asc": true, "nullsFirst": false, @@ -14645,7 +14704,7 @@ "with": "", "method": "btree", "concurrently": false, - "name": "gcp_connectors_org_project_idx", + "name": "gcp_connectors_org_scope_idx", "entityType": "indexes", "schema": "public", "table": "gcp_connectors" diff --git a/packages/db/src/schema/gcp-connectors.ts b/packages/db/src/schema/gcp-connectors.ts index 35cc69730f..5e9d264755 100644 --- a/packages/db/src/schema/gcp-connectors.ts +++ b/packages/db/src/schema/gcp-connectors.ts @@ -1,14 +1,21 @@ -import type { GcpConnectorId, GcpProjectId, OrgId } from "@maple/domain" -import { pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" +import type { GcpConnectorId, GcpProjectId, GcpResourceNumber, GcpScopeType, OrgId } from "@maple/domain" +import { boolean, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" -// One row per connected Google Cloud project. The ingest gateway's `/v1/logpush/gcp/...` receiver -// authenticates a push by `id` + `secret_hash` and writes `last_received_at` / `last_error`. +// One row per connected Google Cloud scope: a project, or everything under a folder or +// organization. `project_id` is the host project, where Maple's own resources live (Pub/Sub topic, +// subscription, service account); for a project scope it equals `scope_id`. The ingest gateway's +// `/v1/logpush/gcp/...` receiver authenticates a push by `id` + `secret_hash` and writes +// `last_received_at` / `last_error`. `metrics_enabled` covers metrics and resource collection. export const gcpConnectors = pgTable( "gcp_connectors", { id: text("id").$type().notNull().primaryKey(), orgId: text("org_id").$type().notNull(), + scopeType: text("scope_type").$type().notNull(), + scopeId: text("scope_id").$type().notNull(), projectId: text("project_id").$type().notNull(), + logsEnabled: boolean("logs_enabled").notNull().default(true), + metricsEnabled: boolean("metrics_enabled").notNull().default(false), secretCiphertext: text("secret_ciphertext").notNull(), secretIv: text("secret_iv").notNull(), secretTag: text("secret_tag").notNull(), @@ -20,7 +27,7 @@ export const gcpConnectors = pgTable( updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }).notNull(), }, (table) => [ - uniqueIndex("gcp_connectors_org_project_idx").on(table.orgId, table.projectId), + uniqueIndex("gcp_connectors_org_scope_idx").on(table.orgId, table.scopeType, table.scopeId), uniqueIndex("gcp_connectors_secret_hash_unique").on(table.secretHash), ], ) diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index 3b992896ed..11bfa1083b 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -47,7 +47,8 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/ErrorIssuePullRequestNotFoundError", "@maple/http/errors/ErrorIssueTransitionError", "@maple/http/errors/ErrorValidationError", - "@maple/http/errors/GcpProjectAlreadyConnectedError", + "@maple/http/errors/GcpMetricsUnavailableError", + "@maple/http/errors/GcpScopeAlreadyConnectedError", "@maple/http/errors/IncidentTriageUnauthorizedError", "@maple/http/errors/IngestAttributeMappingForbiddenError", "@maple/http/errors/IngestAttributeMappingNotFoundError", diff --git a/packages/domain/src/http/integrations.ts b/packages/domain/src/http/integrations.ts index 59af5eaf45..fd15c40183 100644 --- a/packages/domain/src/http/integrations.ts +++ b/packages/domain/src/http/integrations.ts @@ -1,6 +1,13 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" import { Schema } from "effect" -import { ExternalUserId, GcpProjectId, ScrapeTargetId, UserId } from "../primitives" +import { + ExternalUserId, + GcpProjectId, + GcpResourceNumber, + GcpScopeType, + ScrapeTargetId, + UserId, +} from "../primitives" import { Authorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" import { PrReviewListItem, PrReviewOrgSettings, PrReviewRepositoryConfig } from "./pr-review" @@ -1043,24 +1050,42 @@ export class IntegrationsNotFoundError extends HttpTaggedError()( - "@maple/http/errors/GcpProjectAlreadyConnectedError", +/** The organization already has a connector for this Google Cloud project, folder or organization. */ +export class GcpScopeAlreadyConnectedError extends HttpTaggedError()( + "@maple/http/errors/GcpScopeAlreadyConnectedError", { - projectId: GcpProjectId, + scopeType: GcpScopeType, + scopeId: Schema.Union([GcpProjectId, GcpResourceNumber]), message: Schema.String, }, { status: 409, - code: "gcp_project_already_connected", - title: "Project already connected", - param: "project_id", + code: "gcp_scope_already_connected", + title: "Already connected", + param: "scope_id", retry: "never", recovery: "fix_request", exposure: "public_message", }, ) {} +/** Metrics were requested on a deployment that has no Google identity to collect them with. */ +export class GcpMetricsUnavailableError extends HttpTaggedError()( + "@maple/http/errors/GcpMetricsUnavailableError", + { + message: Schema.String, + }, + { + status: 409, + code: "gcp_metrics_unavailable", + title: "Metrics collection is not available", + param: "metrics_enabled", + retry: "never", + recovery: "none", + exposure: "public_message", + }, +) {} + export type IntegrationHttpError = | IntegrationsForbiddenError | IntegrationsConfigurationError diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts index 813711324a..ee6f1bc98a 100644 --- a/packages/domain/src/http/v2/integrations-gcp.ts +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -1,10 +1,12 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api" import { Schema } from "effect" -import { GcpConnectorId, GcpProjectId } from "../../primitives" +import { GcpConnectorId, GcpProjectId, GcpResourceNumber, GcpScopeType } from "../../primitives" import { - GcpProjectAlreadyConnectedError, + GcpMetricsUnavailableError, + GcpScopeAlreadyConnectedError, IntegrationsNotFoundError, IntegrationsPersistenceError, + IntegrationsValidationError, } from "../integrations" import { AuthorizationV2 } from "./auth" import { wireExample, Timestamp } from "./envelopes" @@ -12,9 +14,11 @@ import { V2InsufficientPermissions } from "./errors" import { publicErrors } from "./public-error" import { PublicId, PublicIdPrefixes } from "./public-id" -// Google Cloud integration. No OAuth: an org registers a connector per Google Cloud project and -// its owner runs the generated `gcloud` script, which routes the project's logs through Pub/Sub -// to the ingest gateway. Public v2 from the start so scripted setups can use a scoped API key. +// Google Cloud integration. No OAuth: an org registers a connector for a project, a folder or a +// whole organization, and an administrator runs the generated `gcloud` script. The script routes +// logs through Pub/Sub to the ingest gateway and lets Maple read metrics and the resource +// inventory, each only if the connector opted in. Public v2 from the start so scripted setups can +// use a scoped API key. /** `gcpc_…` public ID ⇄ internal `GcpConnectorId` (raw UUID). */ export const GcpConnectorPublicId = PublicId(PublicIdPrefixes.gcpConnector, GcpConnectorId) @@ -22,7 +26,11 @@ export const GcpConnectorPublicId = PublicId(PublicIdPrefixes.gcpConnector, GcpC const connectorExample = { id: "gcpc_YofPTrK9782DWwcnXhpcCw", object: "gcp_connector", - project_id: "acme-prod", + scope_type: "organization", + scope_id: "123456789012", + project_id: "acme-observability", + logs_enabled: true, + metrics_enabled: true, created_at: "2026-10-01T12:00:00.000Z", last_log_received_at: "2026-10-08T09:12:00.000Z", last_log_error: null, @@ -33,14 +41,33 @@ export const V2GcpConnector = Schema.Struct({ object: Schema.Literal("gcp_connector").annotate({ description: 'The object type — always `"gcp_connector"`.', }), + scope_type: GcpScopeType.annotate({ + description: + "What the connector covers: one `project`, or every project under a `folder` or `organization`, including projects created later.", + examples: ["organization"], + }), + scope_id: Schema.String.annotate({ + description: "The project ID, or the numeric folder or organization ID, named by `scope_type`.", + examples: ["123456789012"], + }), project_id: Schema.String.annotate({ - description: "The Google Cloud project ID this connector receives from.", - examples: ["acme-prod"], + description: + "The host project: where the setup script creates Maple's own resources (Pub/Sub topic, subscription, service account). Equal to `scope_id` for a `project` scope.", + examples: ["acme-observability"], + }), + logs_enabled: Schema.Boolean.annotate({ + description: "Whether the setup script forwards logs to Maple.", + examples: [true], + }), + metrics_enabled: Schema.Boolean.annotate({ + description: + "Whether the setup script lets Maple read Cloud Monitoring metrics and the Cloud Asset resource inventory.", + examples: [true], }), created_at: Timestamp.annotate({ description: "When the connector was created." }), last_log_received_at: Schema.NullOr(Timestamp).annotate({ description: - "When Maple last accepted a log push from this project, or `null` before the first one. Stays `null` until the setup script has run.", + "When Maple last accepted a log push from this connector, or `null` before the first one. Stays `null` until the setup script has run.", }), last_log_error: Schema.NullOr(Schema.String).annotate({ description: "Why the most recent log push was rejected, or `null`.", @@ -49,7 +76,7 @@ export const V2GcpConnector = Schema.Struct({ }).annotate({ identifier: "GcpConnector", title: "Google Cloud connector", - description: "One connected Google Cloud project.", + description: "One connected Google Cloud project, folder or organization.", examples: [wireExample(connectorExample)], }) export type V2GcpConnector = Schema.Schema.Type @@ -60,7 +87,7 @@ export const V2GcpIntegration = Schema.Struct({ }), metrics_available: Schema.Boolean.annotate({ description: - "Whether this Maple deployment can collect Cloud Monitoring metrics. When `false` connectors are logs-only and the setup script omits the metrics steps.", + "Whether this Maple deployment can collect metrics and resources from Google Cloud. When `false`, `metrics_enabled` cannot be turned on.", examples: [true], }), connectors: Schema.Array(V2GcpConnector).annotate({ @@ -70,30 +97,86 @@ export const V2GcpIntegration = Schema.Struct({ identifier: "GcpIntegration", title: "Google Cloud integration status", description: - 'The Google Cloud integration state for the authenticated organization. Note: `connectors` is **not** the standard `{ object: "list", data, has_more, next_cursor }` envelope: an organization connects a handful of projects and they are returned whole next to `metrics_available`.', + 'The Google Cloud integration state for the authenticated organization. Note: `connectors` is **not** the standard `{ object: "list", data, has_more, next_cursor }` envelope: an organization has a handful of connectors and they are returned whole next to `metrics_available`.', examples: [ wireExample({ object: "gcp_integration", metrics_available: true, connectors: [connectorExample] }), ], }) export type V2GcpIntegration = Schema.Schema.Type -export const V2GcpCreateConnectorRequest = Schema.Struct({ - project_id: GcpProjectId.annotate({ - description: - "The Google Cloud project ID (not its name or number): 6 to 30 lowercase letters, digits and hyphens, starting with a letter.", - }), -}).annotate({ +const logsEnabledField = Schema.optionalKey(Schema.Boolean).annotate({ + description: "Forward logs to Maple.", + examples: [true], +}) +const metricsEnabledField = Schema.optionalKey(Schema.Boolean).annotate({ + description: + "Let Maple read metrics and the resource inventory. Rejected when the status reports `metrics_available: false`.", + examples: [false], +}) + +export const V2GcpCreateConnectorRequest = Schema.Union([ + Schema.Struct({ + scope_type: Schema.Literal("project"), + scope_id: GcpProjectId.annotate({ description: "The project ID (not its name or number)." }), + project_id: Schema.optionalKey(GcpProjectId).annotate({ + description: "A project is its own host project: omit this, or repeat `scope_id`.", + }), + logs_enabled: logsEnabledField, + metrics_enabled: metricsEnabledField, + }) + .check( + // Returns the message: a filter's `description` is not what surfaces on failure. + Schema.makeFilter( + (value) => + value.project_id === undefined || + value.project_id === value.scope_id || + "project_id must be omitted or equal scope_id when scope_type is project", + ), + ) + .annotate({ identifier: "GcpCreateProjectConnector", title: "Project connector" }), + Schema.Struct({ + scope_type: Schema.Literals(["folder", "organization"]), + scope_id: GcpResourceNumber.annotate({ description: "The numeric folder or organization ID." }), + project_id: GcpProjectId.annotate({ + description: + "The host project for Maple's own resources (Pub/Sub topic, subscription, service account): a project ID, usually a project inside the folder or organization.", + }), + logs_enabled: logsEnabledField, + metrics_enabled: metricsEnabledField, + }).annotate({ identifier: "GcpCreateAggregatedConnector", title: "Folder or organization connector" }), +]).annotate({ identifier: "GcpCreateConnectorRequest", title: "Google Cloud connector create request", - description: "The project to connect. Connect several projects by creating one connector each.", - examples: [wireExample({ project_id: "acme-prod" })], + description: + "What to connect, discriminated on `scope_type`. `logs_enabled` defaults to `true` and `metrics_enabled` to `false`; at least one must be on.", + examples: [ + wireExample({ scope_type: "project", scope_id: "acme-prod" }), + wireExample({ + scope_type: "organization", + scope_id: "123456789012", + project_id: "acme-observability", + metrics_enabled: true, + }), + ], }) export type V2GcpCreateConnectorRequest = Schema.Schema.Type +export const V2GcpUpdateConnectorRequest = Schema.Struct({ + logs_enabled: logsEnabledField, + metrics_enabled: metricsEnabledField, +}).annotate({ + identifier: "GcpUpdateConnectorRequest", + title: "Google Cloud connector update request", + description: + "What the connector collects; omitted fields are unchanged. At least one must stay on: delete the connector to disconnect.", + examples: [wireExample({ metrics_enabled: true })], +}) +export type V2GcpUpdateConnectorRequest = Schema.Schema.Type + export const V2GcpSetupScriptsRequest = Schema.Struct({ exclude_gke_container_logs: Schema.optionalKey(Schema.Boolean).annotate({ description: - "Leave GKE container logs out of the log filter. Set it when the project's pods already ship their logs to Maple through an OpenTelemetry collector, so they are not ingested twice. Defaults to `false`.", + "Leave GKE container logs out of the log filter. Set it when pods already ship their logs to Maple through an OpenTelemetry collector, so they are not ingested twice. Defaults to `false`.", examples: [false], }), }).annotate({ @@ -110,7 +193,7 @@ export const V2GcpSetupScripts = Schema.Struct({ }), setup_script: Schema.String.annotate({ description: - "A bash script for Cloud Shell, run by a project owner. It embeds the connector's push secret, so treat it as a credential. Safe to re-run; the log filter is an editable variable at the top.", + "A bash script for Cloud Shell. It sets up what the connector has enabled and removes what it has not, so re-run it after changing `logs_enabled` or `metrics_enabled`. With logs enabled it embeds the connector's push secret: treat it as a credential.", examples: ["#!/usr/bin/env bash\n…"], }), cleanup_script: Schema.String.annotate({ @@ -120,7 +203,7 @@ export const V2GcpSetupScripts = Schema.Struct({ }).annotate({ identifier: "GcpSetupScripts", title: "Google Cloud setup scripts", - description: "The scripts that connect and disconnect one Google Cloud project.", + description: "The scripts that connect and disconnect one Google Cloud project, folder or organization.", examples: [ wireExample({ object: "gcp_connector.setup_scripts", @@ -141,13 +224,13 @@ export const V2GcpConnectorDeleteResponse = Schema.Struct({ }), cleanup_script: Schema.String.annotate({ description: - "A bash script that removes the resources the setup script created. Maple has no write access to the project, so the project owner runs it.", + "A bash script that removes the resources the setup script created. Maple has no write access to Google Cloud, so an administrator runs it.", examples: ["#!/usr/bin/env bash\n…"], }), }).annotate({ identifier: "GcpConnectorDeleteResponse", title: "Google Cloud connector delete response", - description: "Confirmation that a connector was deleted, with the script that cleans up the project.", + description: "Confirmation that a connector was deleted, with the script that cleans up Google Cloud.", examples: [ wireExample({ id: connectorExample.id, @@ -159,11 +242,14 @@ export const V2GcpConnectorDeleteResponse = Schema.Struct({ }) export type V2GcpConnectorDeleteResponse = Schema.Schema.Type -const [projectAlreadyConnected, connectorNotFound, integrationPersistence] = publicErrors( - GcpProjectAlreadyConnectedError, - IntegrationsNotFoundError, - IntegrationsPersistenceError, -) +const [scopeAlreadyConnected, metricsUnavailable, validation, connectorNotFound, integrationPersistence] = + publicErrors( + GcpScopeAlreadyConnectedError, + GcpMetricsUnavailableError, + IntegrationsValidationError, + IntegrationsNotFoundError, + IntegrationsPersistenceError, + ) export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration") .add( @@ -175,7 +261,7 @@ export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration identifier: "getGcpIntegration", summary: "Retrieve Google Cloud integration status", description: - "Returns the organization's connected Google Cloud projects with the time and outcome of their latest log push, and whether metrics collection is available. Requires the `integrations:read` scope.", + "Returns the organization's Google Cloud connectors with the time and outcome of their latest log push, and whether metrics collection is available. Requires the `integrations:read` scope.", }), ), ) @@ -183,13 +269,40 @@ export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration HttpApiEndpoint.post("createConnector", "/connectors", { payload: V2GcpCreateConnectorRequest, success: V2GcpConnector, - error: [V2InsufficientPermissions.schema, projectAlreadyConnected, integrationPersistence], + error: [ + V2InsufficientPermissions.schema, + scopeAlreadyConnected, + metricsUnavailable, + validation, + integrationPersistence, + ], }).annotateMerge( OpenApi.annotations({ identifier: "createGcpConnector", - summary: "Connect a Google Cloud project", + summary: "Connect a Google Cloud project, folder or organization", + description: + "Registers a connector for one project, or for every project under a folder or organization. Nothing reaches Maple until an administrator runs the script from `setup_scripts`. A project, folder or organization can be connected once per Maple organization. Requires an org-admin role and the `integrations:write` scope.", + }), + ), + ) + .add( + HttpApiEndpoint.patch("updateConnector", "/connectors/:id", { + params: { id: GcpConnectorPublicId }, + payload: V2GcpUpdateConnectorRequest, + success: V2GcpConnector, + error: [ + V2InsufficientPermissions.schema, + connectorNotFound, + metricsUnavailable, + validation, + integrationPersistence, + ], + }).annotateMerge( + OpenApi.annotations({ + identifier: "updateGcpConnector", + summary: "Change what a connector collects", description: - "Registers a connector for one Google Cloud project. Nothing reaches Maple until the project owner runs the script from `setup_scripts`. A project can be connected once per organization. Requires an org-admin role and the `integrations:write` scope.", + "Turns log forwarding or metrics and resource collection on or off. Maple cannot change Google Cloud itself: re-run the script from `setup_scripts` afterwards, which sets up what is now enabled and removes what is not. Requires an org-admin role and the `integrations:write` scope.", }), ), ) @@ -216,9 +329,9 @@ export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration }).annotateMerge( OpenApi.annotations({ identifier: "deleteGcpConnector", - summary: "Disconnect a Google Cloud project", + summary: "Disconnect a Google Cloud project, folder or organization", description: - "Deletes the connector, after which Maple rejects the project's log pushes. Already-ingested data is unaffected. The response carries the cleanup script to run in the project. Requires an org-admin role and the `integrations:write` scope.", + "Deletes the connector, after which Maple rejects its log pushes and stops reading its metrics. Already-ingested data is unaffected. The response carries the cleanup script to run in Google Cloud. Requires an org-admin role and the `integrations:write` scope.", }), ), ) @@ -228,6 +341,6 @@ export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration OpenApi.annotations({ title: "Google Cloud Integration", description: - "Connect Google Cloud projects to your organization: register a connector per project, fetch the `gcloud` scripts that route its logs to Maple, and see whether logs are arriving.", + "Connect Google Cloud projects, folders or organizations: register a connector, choose whether it forwards logs and whether Maple may read metrics and resources, fetch the `gcloud` scripts that set it up, and see whether logs are arriving.", }), ) {} diff --git a/packages/domain/src/http/v2/openapi.test.ts b/packages/domain/src/http/v2/openapi.test.ts index 8bcc8c70df..4b153585f8 100644 --- a/packages/domain/src/http/v2/openapi.test.ts +++ b/packages/domain/src/http/v2/openapi.test.ts @@ -171,6 +171,7 @@ describe("MapleApiV2 OpenAPI", () => { "PATCH /v2/attribute_mappings/{id}", "PATCH /v2/dashboards/{id}", "PATCH /v2/integrations/chat_workspaces/{id}", + "PATCH /v2/integrations/gcp/connectors/{id}", "PATCH /v2/scrape_targets/{id}", "POST /v2/agent_feedback", "POST /v2/alerts/destinations", diff --git a/packages/primitives/src/index.ts b/packages/primitives/src/index.ts index cce44eb412..c2a3dfd0b7 100644 --- a/packages/primitives/src/index.ts +++ b/packages/primitives/src/index.ts @@ -111,6 +111,17 @@ export const GcpProjectId = Schema.String.check(Schema.isPattern(/^[a-z][a-z0-9- ) export type GcpProjectId = Schema.Schema.Type +/** The numeric id of a Google Cloud folder or organization. */ +export const GcpResourceNumber = Schema.String.check(Schema.isPattern(/^[0-9]{1,20}$/u)).pipe( + Schema.brand("@maple/GcpResourceNumber"), + Schema.annotate({ identifier: "@maple/GcpResourceNumber", title: "Google Cloud Resource Number" }), +) +export type GcpResourceNumber = Schema.Schema.Type + +/** What a Google Cloud connector covers: one project, or everything under a folder or organization. */ +export const GcpScopeType = Schema.Literals(["project", "folder", "organization"]) +export type GcpScopeType = Schema.Schema.Type + export const AlertDestinationId = MapleUuidId("@maple/AlertDestinationId", "Alert Destination ID") export type AlertDestinationId = Schema.Schema.Type From ab4afb0d88860a8157ba28cc58994fe58e0fdfc0 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 19:38:36 +0200 Subject: [PATCH 05/21] fix(integrations): make Google Cloud teardown report what it could not check Script review: removal no longer treats a failed lookup as an absent resource, a failed role removal ends the script with an error, the role comments say what Cloud Asset Viewer can read, and the token-creator grant names the organization policy that can reject it. --- .../integrations/gcp/setup-scripts.test.ts | 61 +++++++++++++++---- .../integrations/gcp/setup-scripts.ts | 61 +++++++++++++------ 2 files changed, 93 insertions(+), 29 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index d8f750dce0..e5524dfc5f 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -62,13 +62,19 @@ const setupInput = ( excludeGkeContainerLogs: false, }) -// A `gcloud` that records its arguments. `describe` reports "not found" unless the test says -// the resources exist, and the sink's writer identity is the one value a script reads back. +// A `gcloud` that records its commands, and each argument on its own line so a value split by +// bad quoting shows. `describe` answers as the test says: found, missing, or denied. The sink's +// writer identity is the one value a script reads back. const FAKE_GCLOUD = `#!/usr/bin/env bash echo "$*" >> "$GCLOUD_LOG" +printf "%s\\n" "$@" >> "$GCLOUD_LOG.args" case "$*" in *"sinks describe"*"--format"*) echo "${WRITER}" ;; - *" describe "*) [ "$GCLOUD_EXISTS" = 1 ] || exit 1 ;; + *" describe "*) + case "$GCLOUD_DESCRIBE" in + missing) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; + denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; + esac ;; *"remove-iam-policy-binding"*) exit "$GCLOUD_REMOVE_BINDING_STATUS" ;; esac ` @@ -79,7 +85,10 @@ afterEach(() => { }) /** Run a rendered script in bash against the fake gcloud; returns the commands it issued. */ -const run = (script: string, options: { exists?: boolean; removeBindingStatus?: number } = {}) => { +const run = ( + script: string, + options: { describe?: "found" | "missing" | "denied"; removeBindingStatus?: number } = {}, +) => { const dir = mkdtempSync(join(tmpdir(), "maple-gcp-script-")) tempDirs.push(dir) writeFileSync(join(dir, "gcloud"), FAKE_GCLOUD) @@ -93,7 +102,7 @@ const run = (script: string, options: { exists?: boolean; removeBindingStatus?: ...process.env, PATH: `${dir}:${process.env.PATH}`, GCLOUD_LOG: log, - GCLOUD_EXISTS: options.exists === true ? "1" : "0", + GCLOUD_DESCRIBE: options.describe ?? "missing", GCLOUD_REMOVE_BINDING_STATUS: String(options.removeBindingStatus ?? 0), }, }) @@ -101,7 +110,9 @@ const run = (script: string, options: { exists?: boolean; removeBindingStatus?: dir, status: result.status, stdout: result.stdout, + stderr: result.stderr, commands: existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : [], + args: existsSync(`${log}.args`) ? readFileSync(`${log}.args`, "utf8").split("\n") : [], } } @@ -125,7 +136,7 @@ const metricsSetupCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, `projects add-iam-policy-binding acme-host --member=serviceAccount:${ACCOUNT} --role=roles/serviceusage.serviceUsageConsumer --condition=None`, - `iam service-accounts add-iam-policy-binding ${ACCOUNT} --project=acme-host --member=serviceAccount:${MAPLE_ACCOUNT} --role=roles/iam.serviceAccountTokenCreator`, + `iam service-accounts add-iam-policy-binding ${ACCOUNT} --project=acme-host --member=serviceAccount:${MAPLE_ACCOUNT} --role=roles/iam.serviceAccountTokenCreator --condition=None`, ] const logsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ @@ -160,7 +171,7 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { it("with logs only, sets up logs and removes an earlier metrics setup", () => { const script = renderGcpSetupScript(setupInput(scopeType, { logs: true, metrics: false })) expect(script).not.toContain(MAPLE_ACCOUNT) - const { status, commands } = run(script, { exists: true }) + const { status, commands } = run(script, { describe: "found" }) expect(status).toBe(0) // Everything exists already, so the log pipeline is updated in place. expect(commands.filter((command) => command.includes(" create "))).toEqual([]) @@ -177,7 +188,7 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { const script = renderGcpSetupScript(setupInput(scopeType, { logs: false, metrics: true })) expect(script).not.toContain(SECRET) expect(script).not.toContain("LOG_FILTER") - const { status, commands } = run(script, { exists: true }) + const { status, commands } = run(script, { describe: "found" }) expect(status).toBe(0) expect(commands.slice(0, logsRemovalCommands(scope).length)).toEqual(logsRemovalCommands(scope)) expect(commands).not.toContainEqual(expect.stringContaining("pubsub.googleapis.com logging")) @@ -189,7 +200,7 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { it("cleans up everything, and touches nothing when nothing exists", () => { const script = renderGcpCleanupScript(scope.target) expect(script).not.toContain("secret") - const existing = run(script, { exists: true }) + const existing = run(script, { describe: "found" }) expect(existing.status).toBe(0) expect(existing.commands).toEqual([...logsRemovalCommands(scope), ...metricsRemovalCommands(scope)]) @@ -215,11 +226,39 @@ describe("renderGcpSetupScript", () => { it("still deletes the service account when one of its role bindings is already gone", () => { const { status, commands } = run(renderGcpCleanupScript(SCOPES.folder.target), { - exists: true, + describe: "found", removeBindingStatus: 1, }) - expect(status).toBe(0) + // Deleted, and reported: the failed removal may have left a binding behind. expect(commands.at(-1)).toBe(`iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`) + expect(status).toBe(1) + }) + + it("does not take a resource it may not look at for one that is gone", () => { + const cleanup = run(renderGcpCleanupScript(SCOPES.organization.target), { describe: "denied" }) + expect(cleanup.commands.every((command) => command.includes(" describe "))).toBe(true) + expect(cleanup.stderr).toContain("PERMISSION_DENIED") + expect(cleanup.stderr).toContain("Not everything could be removed") + expect(cleanup.status).toBe(1) + + // Setting up still works: a failed look falls through to the create, which reports for itself. + const setup = run(renderGcpSetupScript(setupInput("organization", { logs: true, metrics: true })), { + describe: "denied", + }) + expect(setup.status).toBe(0) + }) + + it("hands gcloud each value as one argument", () => { + const { args } = run(renderGcpSetupScript(setupInput("folder", { logs: true, metrics: true }))) + expect(args).toEqual( + expect.arrayContaining([ + `--log-filter=${gcpLogFilter(false)}`, + `--push-endpoint=${PUSH_ENDPOINT}`, + "--display-name=Maple metrics and resource reader", + "--format=value(writerIdentity)", + `--member=${WRITER}`, + ]), + ) }) it("excludes data-access audit logs and health-check probes by default, GKE containers on request", () => { diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index cb37f7e6bd..b643d146a5 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -1,10 +1,5 @@ import { gcpConnectorResourceNames } from "@maple/domain/gcp" -import type { - GcpConnectorId, - GcpProjectId, - GcpResourceNumber, - GcpScopeType, -} from "@maple/domain/primitives" +import type { GcpConnectorId, GcpProjectId, GcpResourceNumber, GcpScopeType } from "@maple/domain/primitives" /** Single-quote a value for bash. Every value that reaches a script goes through here. */ const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` @@ -110,17 +105,42 @@ gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher >/dev/null ` +// Removal must not take "could not look" for "not there": a sink the caller may not see would +// otherwise be left routing into a topic the next step deletes. +const REMOVAL_HELPERS = ` +# exists : whether a resource is there. NOT_FOUND, or an API that was +# never enabled, means it is not. Any other failure (a missing permission, for example) is shown +# and makes this script end with an error, because the resource may still be there. +INCOMPLETE=0 +exists() { + local output + if output="$("$@" 2>&1)"; then return 0; fi + case "$output" in + *NOT_FOUND* | *"has not been used"* | *"is disabled"*) ;; + *) echo "$output" >&2; INCOMPLETE=1 ;; + esac + return 1 +} +` + +const REMOVAL_CHECK = ` +if [ "$INCOMPLETE" = 1 ]; then + echo "Not everything could be removed. See the errors above, then re-run." >&2 + exit 1 +fi +` + const logsRemoval = (scopeType: GcpScopeType): string => ` # ---- Logs: off. Remove what an earlier run created. ---- # The Logs Writer grant on the host project stays: the sink wrote as the logging service agent # of the ${scopeType}, an identity every other sink there shares. -if gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} >/dev/null 2>&1; then +if exists gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then gcloud logging sinks delete "$SINK" ${SCOPES[scopeType].sink} --quiet fi -if gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID" >/dev/null 2>&1; then +if exists gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" --quiet fi -if gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID" >/dev/null 2>&1; then +if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" --quiet fi ` @@ -156,6 +176,8 @@ retry() { # Read-only roles on the ${scopeType}${scopeType === "project" ? "" : ", inherited by every project under it"}: # roles/monitoring.viewer read metrics (monitoring.timeSeries.list) # roles/cloudasset.viewer list resources (cloudasset.assets.searchAllResources) +# These are the narrowest predefined roles for the two calls. Both are read-only, and broader +# than the calls: Cloud Asset Viewer can also read resource metadata and IAM policies. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do retry ${SCOPES[scopeType].iam} add-iam-policy-binding "$SCOPE_ID" \\ ${serviceAccountMember} --role="$ROLE" --condition=None >/dev/null @@ -166,21 +188,23 @@ retry gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None >/dev/null # Maple's own account may mint short-lived tokens for this one service account, nothing else. +# If an organization policy rejects this (domain-restricted sharing, +# constraints/iam.allowedPolicyMemberDomains), allow Maple's account there and re-run. retry gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" \\ - --member="serviceAccount:$MAPLE_SERVICE_ACCOUNT" --role=roles/iam.serviceAccountTokenCreator >/dev/null + --member="serviceAccount:$MAPLE_SERVICE_ACCOUNT" --role=roles/iam.serviceAccountTokenCreator \\ + --condition=None >/dev/null ` const metricsRemoval = (scopeType: GcpScopeType): string => ` # ---- Metrics and resources: off. Remove what an earlier run created. ---- -if gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" >/dev/null 2>&1; then - # Role bindings first, or the deleted account would stay listed in the IAM policy. A binding - # that is already gone is reported and skipped. +if exists gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID"; then + # Role bindings first, or the deleted account would stay listed in the IAM policy. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do ${SCOPES[scopeType].iam} remove-iam-policy-binding "$SCOPE_ID" \\ - ${serviceAccountMember} --role="$ROLE" --condition=None >/dev/null || true + ${serviceAccountMember} --role="$ROLE" --condition=None >/dev/null || INCOMPLETE=1 done gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ - ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None >/dev/null || true + ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None >/dev/null || INCOMPLETE=1 gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet fi ` @@ -215,6 +239,7 @@ LOG_FILTER=${sh(gcpLogFilter(input.excludeGkeContainerLogs))} section: logsSetup(input.scopeType, input.pushEndpoint), } : { note: "", filter: "", section: logsRemoval(input.scopeType) } + const removes = !input.logsEnabled || mapleAccount === undefined return `#!/usr/bin/env bash # Maple: connect a Google Cloud ${input.scopeType} to Maple. # Logs: ${input.logsEnabled ? "on" : "off"} @@ -226,11 +251,11 @@ ${SCOPES[input.scopeType].needs} ${logs.note}set -euo pipefail ${logs.filter} ${variables(input)} -${logs.section}${ +${removes ? REMOVAL_HELPERS : ""}${logs.section}${ mapleAccount === undefined ? metricsRemoval(input.scopeType) : metricsSetup(input.scopeType, mapleAccount) - } + }${removes ? REMOVAL_CHECK : ""} echo "Maple setup complete." ` } @@ -246,6 +271,6 @@ ${SCOPES[target.scopeType].needs} set -euo pipefail ${variables(target)} -${logsRemoval(target.scopeType)}${metricsRemoval(target.scopeType)} +${REMOVAL_HELPERS}${logsRemoval(target.scopeType)}${metricsRemoval(target.scopeType)}${REMOVAL_CHECK} echo "Maple cleanup complete." ` From c17da5413903cce2c77114ab2baf2e5d32357771 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 20:01:26 +0200 Subject: [PATCH 06/21] fix(integrations): lock the connector row while its switches change Bot review: two concurrent updates could each turn the other capability off; the update now reads and writes under a row lock. A disabled API no longer counts as an absent log sink during teardown, only as absent host-project resources. --- .../integrations/GcpConnectorService.ts | 70 +++++++++++-------- .../integrations/gcp/setup-scripts.test.ts | 12 +++- .../integrations/gcp/setup-scripts.ts | 13 ++-- 3 files changed, 59 insertions(+), 36 deletions(-) diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index 57bf395de4..3150bf2437 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -135,27 +135,22 @@ export class GcpConnectorService extends Context.Service => { + ): GcpCapabilityError | undefined => { if (!flags.logsEnabled && !flags.metricsEnabled) { - return Effect.fail( - new IntegrationsValidationError({ - message: - "At least one of logs_enabled and metrics_enabled must be on. Delete the connector to disconnect.", - }), - ) + return new IntegrationsValidationError({ + message: + "At least one of logs_enabled and metrics_enabled must be on. Delete the connector to disconnect.", + }) } if (turningMetricsOn && mapleServiceAccountEmail === undefined) { - return Effect.fail( - new GcpMetricsUnavailableError({ - message: - "Metrics and resource collection is not available on this Maple deployment.", - }), - ) + return new GcpMetricsUnavailableError({ + message: "Metrics and resource collection is not available on this Maple deployment.", + }) } - return Effect.void + return undefined } const selectRow = (orgId: OrgId, connectorId: GcpConnectorId) => @@ -196,7 +191,8 @@ export class GcpConnectorService extends Context.Service - db - .update(gcpConnectors) - .set({ ...flags, updatedAt }) - .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) - .returning(), + const outcome = yield* dbExecute((db) => + db.transaction((tx) => + Effect.gen(function* () { + // The row lock keeps one capability on when two updates race. + const [current] = yield* tx + .select() + .from(gcpConnectors) + .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) + .limit(1) + .for("update") + if (current === undefined) return undefined + const flags = { + logsEnabled: patch.logsEnabled ?? current.logsEnabled, + metricsEnabled: patch.metricsEnabled ?? current.metricsEnabled, + } + const invalid = capabilityError(flags, patch.metricsEnabled === true) + if (invalid === undefined) { + yield* tx + .update(gcpConnectors) + .set({ ...flags, updatedAt }) + .where(eq(gcpConnectors.id, connectorId)) + } + return { row: { ...current, ...flags }, invalid } + }), + ), ) - if (row === undefined) return yield* notFound() - return toConnector(row) + if (outcome === undefined) return yield* notFound() + if (outcome.invalid !== undefined) return yield* Effect.fail(outcome.invalid) + return toConnector(outcome.row) }) const scripts = Effect.fn("GcpConnectorService.scripts")(function* ( diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index e5524dfc5f..8ed36ab0e5 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -74,6 +74,7 @@ case "$*" in case "$GCLOUD_DESCRIBE" in missing) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; + api_off) echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2; exit 1 ;; esac ;; *"remove-iam-policy-binding"*) exit "$GCLOUD_REMOVE_BINDING_STATUS" ;; esac @@ -87,7 +88,7 @@ afterEach(() => { /** Run a rendered script in bash against the fake gcloud; returns the commands it issued. */ const run = ( script: string, - options: { describe?: "found" | "missing" | "denied"; removeBindingStatus?: number } = {}, + options: { describe?: "found" | "missing" | "denied" | "api_off"; removeBindingStatus?: number } = {}, ) => { const dir = mkdtempSync(join(tmpdir(), "maple-gcp-script-")) tempDirs.push(dir) @@ -248,6 +249,15 @@ describe("renderGcpSetupScript", () => { expect(setup.status).toBe(0) }) + it("reads a disabled API as nothing to remove in the host project, but not for the sink", () => { + const { status, stderr, commands } = run(renderGcpCleanupScript(SCOPES.organization.target), { + describe: "api_off", + }) + expect(commands.every((command) => command.includes(" describe "))).toBe(true) + expect(stderr.match(/has not been used/g)).toHaveLength(1) + expect(status).toBe(1) + }) + it("hands gcloud each value as one argument", () => { const { args } = run(renderGcpSetupScript(setupInput("folder", { logs: true, metrics: true }))) expect(args).toEqual( diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index b643d146a5..bfda730c0a 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -108,15 +108,18 @@ gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ // Removal must not take "could not look" for "not there": a sink the caller may not see would // otherwise be left routing into a topic the next step deletes. const REMOVAL_HELPERS = ` -# exists : whether a resource is there. NOT_FOUND, or an API that was -# never enabled, means it is not. Any other failure (a missing permission, for example) is shown -# and makes this script end with an error, because the resource may still be there. +# exists : whether a resource is there. NOT_FOUND means it is not, and so +# does a disabled API for the host project's own resources, which cannot exist without it (a +# sink can). Any other failure (a missing permission, for example) is shown and makes this script +# end with an error, because the resource may still be there. INCOMPLETE=0 exists() { local output if output="$("$@" 2>&1)"; then return 0; fi - case "$output" in - *NOT_FOUND* | *"has not been used"* | *"is disabled"*) ;; + case "$2:$output" in + *NOT_FOUND*) ;; + logging:*) echo "$output" >&2; INCOMPLETE=1 ;; + *"has not been used"* | *"is disabled"*) ;; *) echo "$output" >&2; INCOMPLETE=1 ;; esac return 1 From 8dce20f59e066879c9007199c95a25e0d64e6434 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 8 Oct 2026 20:16:36 +0200 Subject: [PATCH 07/21] fix(integrations): never tear down Google Cloud resources on an unclear answer Bot review: the topic stays when the sink lookup failed, the service account stays until every role binding is confirmed gone, and a connector with metrics on is left untouched when the deployment has lost its Google identity instead of having its metrics setup removed. --- .../integrations/gcp/setup-scripts.test.ts | 54 +++++++++++----- .../integrations/gcp/setup-scripts.ts | 62 +++++++++++++------ 2 files changed, 81 insertions(+), 35 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index 8ed36ab0e5..fccb5f3e5a 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -76,7 +76,11 @@ case "$*" in denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; api_off) echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2; exit 1 ;; esac ;; - *"remove-iam-policy-binding"*) exit "$GCLOUD_REMOVE_BINDING_STATUS" ;; + *"remove-iam-policy-binding"*) + case "$GCLOUD_UNBIND" in + absent) echo "ERROR: Policy binding with the specified principal, role, and condition not found!" >&2; exit 1 ;; + denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; + esac ;; esac ` @@ -88,11 +92,15 @@ afterEach(() => { /** Run a rendered script in bash against the fake gcloud; returns the commands it issued. */ const run = ( script: string, - options: { describe?: "found" | "missing" | "denied" | "api_off"; removeBindingStatus?: number } = {}, + options: { + describe?: "found" | "missing" | "denied" | "api_off" + unbind?: "ok" | "absent" | "denied" + gcloud?: string + } = {}, ) => { const dir = mkdtempSync(join(tmpdir(), "maple-gcp-script-")) tempDirs.push(dir) - writeFileSync(join(dir, "gcloud"), FAKE_GCLOUD) + writeFileSync(join(dir, "gcloud"), options.gcloud ?? FAKE_GCLOUD) chmodSync(join(dir, "gcloud"), 0o755) const log = join(dir, "gcloud.log") const result = spawnSync("bash", [], { @@ -104,7 +112,7 @@ const run = ( PATH: `${dir}:${process.env.PATH}`, GCLOUD_LOG: log, GCLOUD_DESCRIBE: options.describe ?? "missing", - GCLOUD_REMOVE_BINDING_STATUS: String(options.removeBindingStatus ?? 0), + GCLOUD_UNBIND: options.unbind ?? "ok", }, }) return { @@ -212,26 +220,42 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { }) describe("renderGcpSetupScript", () => { - it("treats metrics as off when the deployment has no Google identity", () => { + it("leaves an enabled metrics setup alone when the deployment has lost its Google identity", () => { const script = renderGcpSetupScript({ ...setupInput("organization", { logs: true, metrics: true }), mapleServiceAccountEmail: undefined, }) - const { status, commands } = run(script) + expect(script).toContain("Metrics and resources: on, unavailable") + const { status, commands } = run(script, { describe: "found" }) expect(status).toBe(0) - expect(commands).toEqual([ - ...logsSetupCommands(SCOPES.organization), - `iam service-accounts describe ${ACCOUNT} --project=acme-host`, - ]) + expect(commands).not.toContainEqual(expect.stringContaining("service-accounts")) + expect(commands).not.toContainEqual(expect.stringContaining("remove-iam-policy-binding")) }) it("still deletes the service account when one of its role bindings is already gone", () => { - const { status, commands } = run(renderGcpCleanupScript(SCOPES.folder.target), { - describe: "found", - removeBindingStatus: 1, + const script = renderGcpCleanupScript(SCOPES.folder.target) + const absent = run(script, { describe: "found", unbind: "absent" }) + expect(absent.commands.at(-1)).toBe( + `iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`, + ) + expect(absent.status).toBe(0) + + // A binding that could not be removed keeps the account, so a later run can still name it. + const denied = run(script, { describe: "found", unbind: "denied" }) + expect(denied.commands).not.toContainEqual(expect.stringContaining("service-accounts delete")) + expect(denied.status).toBe(1) + }) + + it("keeps the topic when it cannot tell whether the sink is gone", () => { + const sinkDenied = `#!/usr/bin/env bash +echo "$*" >> "$GCLOUD_LOG" +case "$*" in *"sinks describe"*) echo "ERROR: PERMISSION_DENIED" >&2; exit 1 ;; esac +` + const { status, commands } = run(renderGcpCleanupScript(SCOPES.organization.target), { + gcloud: sinkDenied, }) - // Deleted, and reported: the failed removal may have left a binding behind. - expect(commands.at(-1)).toBe(`iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`) + expect(commands).not.toContainEqual(expect.stringContaining("pubsub")) + expect(commands).not.toContainEqual(expect.stringContaining("service-accounts delete")) expect(status).toBe(1) }) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index bfda730c0a..9b12fbf429 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -124,6 +124,16 @@ exists() { esac return 1 } + +# unbind : a binding that is already gone is fine. +unbind() { + local output + if output="$("$@" 2>&1)"; then return 0; fi + case "$output" in + *"not found"*) ;; + *) echo "$output" >&2; INCOMPLETE=1 ;; + esac +} ` const REMOVAL_CHECK = ` @@ -140,11 +150,14 @@ const logsRemoval = (scopeType: GcpScopeType): string => ` if exists gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then gcloud logging sinks delete "$SINK" ${SCOPES[scopeType].sink} --quiet fi -if exists gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then - gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" --quiet -fi -if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then - gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" --quiet +# The topic stays while it is unclear whether a sink still writes to it. +if [ "$INCOMPLETE" = 0 ]; then + if exists gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then + gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" --quiet + fi + if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then + gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" --quiet + fi fi ` @@ -201,21 +214,30 @@ retry gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL const metricsRemoval = (scopeType: GcpScopeType): string => ` # ---- Metrics and resources: off. Remove what an earlier run created. ---- if exists gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID"; then - # Role bindings first, or the deleted account would stay listed in the IAM policy. + # Role bindings first: once the account is deleted they can no longer be removed by name, so + # the account stays until every one of them is gone. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do - ${SCOPES[scopeType].iam} remove-iam-policy-binding "$SCOPE_ID" \\ - ${serviceAccountMember} --role="$ROLE" --condition=None >/dev/null || INCOMPLETE=1 + unbind ${SCOPES[scopeType].iam} remove-iam-policy-binding "$SCOPE_ID" \\ + ${serviceAccountMember} --role="$ROLE" --condition=None done - gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ - ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None >/dev/null || INCOMPLETE=1 - gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet + unbind gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ + ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None + if [ "$INCOMPLETE" = 0 ]; then + gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet + fi fi ` +// Metrics are on for the connector but this deployment has lost its Google identity. Nothing is +// set up and, above all, nothing a customer set up earlier is removed. +const METRICS_UNAVAILABLE = ` +# ---- Metrics and resources: on, but not available on this Maple deployment. Left as is. ---- +` + export interface GcpSetupScriptInput extends GcpScriptTarget { /** The ingest gateway's receiver URL for this connector, including its secret. */ readonly pushEndpoint: string - /** Maple's own Google service account. Undefined: metrics cannot be set up. */ + /** Maple's own Google service account. Undefined: the metrics setup is left untouched. */ readonly mapleServiceAccountEmail: string | undefined readonly logsEnabled: boolean readonly metricsEnabled: boolean @@ -229,7 +251,11 @@ export interface GcpSetupScriptInput extends GcpScriptTarget { * enabled is created or updated, what it has not is removed, so a re-run applies an opt-out. */ export const renderGcpSetupScript = (input: GcpSetupScriptInput): string => { - const mapleAccount = input.metricsEnabled ? input.mapleServiceAccountEmail : undefined + const metrics = !input.metricsEnabled + ? { state: "off", section: metricsRemoval(input.scopeType) } + : input.mapleServiceAccountEmail === undefined + ? { state: "on, unavailable", section: METRICS_UNAVAILABLE } + : { state: "on", section: metricsSetup(input.scopeType, input.mapleServiceAccountEmail) } const logs = input.logsEnabled ? { note: "#\n# Keep this script private: PUSH_ENDPOINT contains this connector's secret.\n", @@ -242,11 +268,11 @@ LOG_FILTER=${sh(gcpLogFilter(input.excludeGkeContainerLogs))} section: logsSetup(input.scopeType, input.pushEndpoint), } : { note: "", filter: "", section: logsRemoval(input.scopeType) } - const removes = !input.logsEnabled || mapleAccount === undefined + const removes = !input.logsEnabled || !input.metricsEnabled return `#!/usr/bin/env bash # Maple: connect a Google Cloud ${input.scopeType} to Maple. # Logs: ${input.logsEnabled ? "on" : "off"} -# Metrics and resources: ${mapleAccount === undefined ? "off" : "on"} +# Metrics and resources: ${metrics.state} # Run in Cloud Shell. Safe to re-run: it sets up what is on and removes what is off. # # You need: @@ -254,11 +280,7 @@ ${SCOPES[input.scopeType].needs} ${logs.note}set -euo pipefail ${logs.filter} ${variables(input)} -${removes ? REMOVAL_HELPERS : ""}${logs.section}${ - mapleAccount === undefined - ? metricsRemoval(input.scopeType) - : metricsSetup(input.scopeType, mapleAccount) - }${removes ? REMOVAL_CHECK : ""} +${removes ? REMOVAL_HELPERS : ""}${logs.section}${metrics.section}${removes ? REMOVAL_CHECK : ""} echo "Maple setup complete." ` } From 8685c6da03a5689f1ef3829380ee0974dcbb7304 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 00:11:23 +0200 Subject: [PATCH 08/21] fix(integrations): look the Google Cloud reader account up in the list when removing it For 30 days after a service account is deleted, describing it answers PERMISSION_DENIED even to an Owner, so a second run of the cleanup script, or of a setup script with metrics off, ended with "Not everything could be removed". The list does not name a deleted account. Setting up after such a delete creates the account again, which the tests now pin. --- .../integrations/gcp/setup-scripts.test.ts | 64 +++++++++++++++++-- .../integrations/gcp/setup-scripts.ts | 24 ++++++- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index fccb5f3e5a..3484bc234f 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -63,13 +63,21 @@ const setupInput = ( }) // A `gcloud` that records its commands, and each argument on its own line so a value split by -// bad quoting shows. `describe` answers as the test says: found, missing, or denied. The sink's -// writer identity is the one value a script reads back. +// bad quoting shows. `describe` answers as the test says: found, missing, or denied, and so does +// the service account list (a missing account is an empty list). The sink's writer identity is +// the one value a script reads back. const FAKE_GCLOUD = `#!/usr/bin/env bash echo "$*" >> "$GCLOUD_LOG" printf "%s\\n" "$@" >> "$GCLOUD_LOG.args" case "$*" in *"sinks describe"*"--format"*) echo "${WRITER}" ;; + *"service-accounts list"*) + case "$GCLOUD_DESCRIBE" in + found) echo "${ACCOUNT}" ;; + missing) echo "Listed 0 items." >&2 ;; + denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; + api_off) echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2; exit 1 ;; + esac ;; *" describe "*) case "$GCLOUD_DESCRIBE" in missing) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; @@ -84,6 +92,17 @@ case "$*" in esac ` +// A service account an earlier run deleted: what Google answers for 30 days, even to an Owner. +const FAKE_GCLOUD_ACCOUNT_DELETED = `#!/usr/bin/env bash +echo "$*" >> "$GCLOUD_LOG" +case "$*" in + *"sinks describe"*"--format"*) echo "${WRITER}" ;; + *"service-accounts describe"*) echo "ERROR: (gcloud) PERMISSION_DENIED: Permission 'iam.serviceAccounts.get' denied on resource (or it may not exist)." >&2; exit 1 ;; + *"service-accounts list"*) echo "Listed 0 items." >&2 ;; + *" describe "*) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; +esac +` + const tempDirs: string[] = [] afterEach(() => { for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }) @@ -157,8 +176,13 @@ const logsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ `pubsub topics delete ${NAME} --project=acme-host --quiet`, ] +const ACCOUNT_LOOKUP = `iam service-accounts list --project=acme-host --filter=email=${ACCOUNT} --format=value(email)` + +/** A command that only looks: a describe, or the service account list. */ +const isLookup = (command: string) => command.includes(" describe ") || command === ACCOUNT_LOOKUP + const metricsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ - `iam service-accounts describe ${ACCOUNT} --project=acme-host`, + ACCOUNT_LOOKUP, `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, `projects remove-iam-policy-binding acme-host --member=serviceAccount:${ACCOUNT} --role=roles/serviceusage.serviceUsageConsumer --condition=None`, @@ -215,7 +239,7 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { const fresh = run(script) expect(fresh.status).toBe(0) - expect(fresh.commands.every((command) => command.includes(" describe "))).toBe(true) + expect(fresh.commands.every(isLookup)).toBe(true) }) }) @@ -261,7 +285,7 @@ case "$*" in *"sinks describe"*) echo "ERROR: PERMISSION_DENIED" >&2; exit 1 ;; it("does not take a resource it may not look at for one that is gone", () => { const cleanup = run(renderGcpCleanupScript(SCOPES.organization.target), { describe: "denied" }) - expect(cleanup.commands.every((command) => command.includes(" describe "))).toBe(true) + expect(cleanup.commands.every(isLookup)).toBe(true) expect(cleanup.stderr).toContain("PERMISSION_DENIED") expect(cleanup.stderr).toContain("Not everything could be removed") expect(cleanup.status).toBe(1) @@ -277,11 +301,39 @@ case "$*" in *"sinks describe"*) echo "ERROR: PERMISSION_DENIED" >&2; exit 1 ;; const { status, stderr, commands } = run(renderGcpCleanupScript(SCOPES.organization.target), { describe: "api_off", }) - expect(commands.every((command) => command.includes(" describe "))).toBe(true) + expect(commands.every(isLookup)).toBe(true) expect(stderr.match(/has not been used/g)).toHaveLength(1) expect(status).toBe(1) }) + it("finishes when the service account was deleted by an earlier run", () => { + const cleanup = run(renderGcpCleanupScript(SCOPES.project.target), { + gcloud: FAKE_GCLOUD_ACCOUNT_DELETED, + }) + expect(cleanup.stdout).toContain("Maple cleanup complete.") + expect(cleanup.status).toBe(0) + expect(cleanup.commands.every(isLookup)).toBe(true) + + // So does a setup script with metrics off, run a second time. + const metricsOff = run(renderGcpSetupScript(setupInput("project", { logs: true, metrics: false })), { + gcloud: FAKE_GCLOUD_ACCOUNT_DELETED, + }) + expect(metricsOff.status).toBe(0) + expect(metricsOff.commands.at(-1)).toBe(ACCOUNT_LOOKUP) + }) + + it("creates the service account again after an earlier run deleted it", () => { + const { status, commands } = run( + renderGcpSetupScript(setupInput("folder", { logs: false, metrics: true })), + { gcloud: FAKE_GCLOUD_ACCOUNT_DELETED }, + ) + expect(status).toBe(0) + // The new account has none of the deleted one's roles: all four grants are made again. + expect(commands.slice(-metricsSetupCommands(SCOPES.folder).length)).toEqual( + metricsSetupCommands(SCOPES.folder), + ) + }) + it("hands gcloud each value as one argument", () => { const { args } = run(renderGcpSetupScript(setupInput("folder", { logs: true, metrics: true }))) expect(args).toEqual( diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 9b12fbf429..3e1718cdff 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -125,6 +125,22 @@ exists() { return 1 } +# listed : whether the list names the +# resource. A list that fails is read like a failed describe above. +listed() { + local output + if output="$("$@" 2>/dev/null)"; then + [ -n "$output" ] + return + fi + output="$("$@" 2>&1 || true)" + case "$output" in + *"has not been used"* | *"is disabled"*) ;; + *) echo "$output" >&2; INCOMPLETE=1 ;; + esac + return 1 +} + # unbind : a binding that is already gone is fine. unbind() { local output @@ -175,6 +191,9 @@ MAPLE_SERVICE_ACCOUNT=${sh(mapleServiceAccountEmail)} gcloud services enable monitoring.googleapis.com cloudasset.googleapis.com \\ iam.googleapis.com iamcredentials.googleapis.com --project="$PROJECT_ID" +# A service account that an earlier run deleted (metrics switched off) fails this describe as +# well and is created again. Google makes that a new account under the same name, with none of +# the old one's roles, so every grant below is made for it. if ! gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" >/dev/null 2>&1; then gcloud iam service-accounts create "$SERVICE_ACCOUNT" --project="$PROJECT_ID" \\ --display-name="Maple metrics and resource reader" @@ -213,7 +232,10 @@ retry gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL const metricsRemoval = (scopeType: GcpScopeType): string => ` # ---- Metrics and resources: off. Remove what an earlier run created. ---- -if exists gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID"; then +# Looked up in the list: for 30 days after a service account is deleted, describing it answers +# PERMISSION_DENIED even to an Owner, which would read as "may still be there" on every re-run. +if listed gcloud iam service-accounts list --project="$PROJECT_ID" \\ + --filter="email=$SERVICE_ACCOUNT_EMAIL" --format='value(email)'; then # Role bindings first: once the account is deleted they can no longer be removed by name, so # the account stays until every one of them is gone. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do From 5b1ecdb85b296c7b79a7cde1c8dc7f0850b08e01 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 00:11:23 +0200 Subject: [PATCH 09/21] fix(integrations): forget the last Google Cloud log push when log forwarding is switched back on No sink exists until the setup script is re-run, so the connection reads "Waiting for the first log" again instead of the push from before the opt-out. --- .../integrations/GcpConnectorService.test.ts | 29 +++++++++++++++++++ .../integrations/GcpConnectorService.ts | 12 ++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts index 9223f38e84..2da7128267 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.test.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -180,6 +180,35 @@ describe("GcpConnectorService", () => { }).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))) }) + it.effect("forgets the last push when log forwarding is switched back on", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const gcp = yield* GcpConnectorService + const connector = yield* gcp.create(orgId, userId, organizationScope) + yield* Effect.promise(() => + executeSql( + testDb, + "UPDATE gcp_connectors SET last_received_at = $1, last_error = $2 WHERE id = $3", + ["2026-10-08T09:12:00.000Z", "payload was not a LogEntry", connector.id], + ), + ) + const lastPush = { + lastLogReceivedAt: Date.parse("2026-10-08T09:12:00.000Z"), + lastLogError: "payload was not a LogEntry", + } + + // Kept while logs stay on, and while they are off. + const stillOn = yield* gcp.update(orgId, connector.id, { logsEnabled: true }) + assert.deepStrictEqual(stillOn, { ...connector, ...lastPush }) + const off = yield* gcp.update(orgId, connector.id, { logsEnabled: false }) + assert.deepStrictEqual(off, { ...connector, ...lastPush, logsEnabled: false }) + + const backOn = yield* gcp.update(orgId, connector.id, { logsEnabled: true }) + assert.deepStrictEqual(backOn, connector) + assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, [connector]) + }).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))) + }) + it.effect("reports what the ingest gateway recorded for the last push", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index 3150bf2437..6cf218c01e 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -258,13 +258,21 @@ export class GcpConnectorService extends Context.Service Date: Fri, 9 Oct 2026 00:35:53 +0200 Subject: [PATCH 10/21] fix(integrations): run the Google Cloud removal lookups without gcloud prompts For an API that is off, gcloud asks whether to enable it and waits for the answer. The lookups capture that question, so it never showed and the script appeared to hang. Without prompts gcloud reports the API as disabled, which the lookups already handle. --- .../services/integrations/gcp/setup-scripts.test.ts | 13 +++++++++++-- .../src/services/integrations/gcp/setup-scripts.ts | 5 +++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index 3484bc234f..a4316e69d9 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -69,6 +69,15 @@ const setupInput = ( const FAKE_GCLOUD = `#!/usr/bin/env bash echo "$*" >> "$GCLOUD_LOG" printf "%s\\n" "$@" >> "$GCLOUD_LOG.args" +# For a disabled API gcloud asks whether to enable it, unless prompts are off. +api_off() { + if [ "$CLOUDSDK_CORE_DISABLE_PROMPTS" != 1 ]; then + echo "API not enabled on project [1]. Would you like to enable and retry? (y/N)" >&2 + exit 1 + fi + echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2 + exit 1 +} case "$*" in *"sinks describe"*"--format"*) echo "${WRITER}" ;; *"service-accounts list"*) @@ -76,13 +85,13 @@ case "$*" in found) echo "${ACCOUNT}" ;; missing) echo "Listed 0 items." >&2 ;; denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; - api_off) echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2; exit 1 ;; + api_off) api_off ;; esac ;; *" describe "*) case "$GCLOUD_DESCRIBE" in missing) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; - api_off) echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2; exit 1 ;; + api_off) api_off ;; esac ;; *"remove-iam-policy-binding"*) case "$GCLOUD_UNBIND" in diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 3e1718cdff..8e2d31d1c4 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -108,6 +108,11 @@ gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ // Removal must not take "could not look" for "not there": a sink the caller may not see would // otherwise be left routing into a topic the next step deletes. const REMOVAL_HELPERS = ` +# For an API that is off, gcloud asks whether to enable it and waits for the answer. The lookups +# below capture that question, so it would never show. Without prompts gcloud reports the API as +# disabled instead. +export CLOUDSDK_CORE_DISABLE_PROMPTS=1 + # exists : whether a resource is there. NOT_FOUND means it is not, and so # does a disabled API for the host project's own resources, which cannot exist without it (a # sink can). Any other failure (a missing permission, for example) is shown and makes this script From 903f2f0709cb932f88fde6168e409e8b03525e4b Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 01:58:42 +0200 Subject: [PATCH 11/21] feat(integrations): Google Cloud setup script that is safe to paste, reports what it applied, and explains every failure The copied text runs the script in a bash process of its own, so a failing step cannot close the Cloud Shell session. The script checks access and permissions before the first change, prints one line per step with Google's answer and a remedy only on failure, keeps an existing sink's log filter unless asked to replace it, and tells Maple what it set up or removed. The connector gains applied_logs_enabled, applied_metrics_enabled and setup_reported_at for that report; setup_scripts takes log_filter in place of exclude_gke_container_logs. --- .../routes/v2/integrations-gcp.http.test.ts | 27 +- .../src/routes/v2/integrations-gcp.http.ts | 5 +- docs/api-v2.md | 2 +- .../integrations/GcpConnectorService.test.ts | 32 +- .../integrations/GcpConnectorService.ts | 27 +- .../integrations/gcp/setup-scripts.test.ts | 684 +++++++++++++++--- .../integrations/gcp/setup-scripts.ts | 684 +++++++++++++----- .../migration.sql | 3 + .../snapshot.json | 41 +- packages/db/src/schema/gcp-connectors.ts | 8 +- .../domain/src/http/v2/integrations-gcp.ts | 54 +- packages/primitives/src/index.ts | 7 + 12 files changed, 1260 insertions(+), 314 deletions(-) rename packages/db/drizzle/{20261008171958_gcp_connectors => 20261008235400_gcp_connectors}/migration.sql (87%) rename packages/db/drizzle/{20261008171958_gcp_connectors => 20261008235400_gcp_connectors}/snapshot.json (99%) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index 6b620097b0..d337006366 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -155,6 +155,9 @@ describe("v2 gcp integration over HTTP", () => { created_at: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T.*Z$/), last_log_received_at: null, last_log_error: null, + applied_logs_enabled: null, + applied_metrics_enabled: null, + setup_reported_at: null, }) // A project is its own host project, and forwards logs only unless asked otherwise. @@ -184,6 +187,8 @@ describe("v2 gcp integration over HTTP", () => { 'gcloud logging sinks delete "$SINK" --organization="$SCOPE_ID"', ), }) + // The connector is gone: its cleanup script has nothing to report to. + expect(deleted.body.cleanup_script).not.toContain("maple_gcp_") const gone = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) expect(gone.status).toBe(404) @@ -204,7 +209,7 @@ describe("v2 gcp integration over HTTP", () => { const path = `${CONNECTORS}/${created.body.id}` const both = await harness.request("POST", `${path}/setup_scripts`, admin.secret, { - exclude_gke_container_logs: true, + log_filter: "exclude_gke_container_logs", }) expect(both.status).toBe(200) expect(Object.keys(both.body).sort()).toEqual(["cleanup_script", "object", "setup_script"]) @@ -215,18 +220,32 @@ describe("v2 gcp integration over HTTP", () => { expect(both.body.setup_script).toContain("PROJECT_ID='acme-host'") expect(both.body.setup_script).toContain("SCOPE_ID='123456789012'") expect(both.body.setup_script).toContain('--folder="$SCOPE_ID" --include-children') - expect(both.body.setup_script).toContain('NOT resource.type="k8s_container"') + expect(both.body.setup_script).toContain( + `LOG_FILTER_MODE='set'\nLOG_FILTER='NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT resource.type="k8s_container"'`, + ) expect(both.body.setup_script).toContain( 'gcloud resource-manager folders add-iam-policy-binding "$SCOPE_ID"', ) - expect(both.body.cleanup_script).not.toContain("maple_gcp_") + // What is copied is the script as a here-document for a bash process of its own. + expect(both.body.setup_script).toMatch(/^ \{ .*\nbash \/dev\/fd\/3 3<<'MAPLE_SETUP_SCRIPT'\n/) + expect(both.body.cleanup_script).toMatch(/\nMAPLE_CLEANUP_SCRIPT\n\}\n$/) + // The cleanup script tells Maple that it ran, with the connector's secret. + expect(both.body.cleanup_script).toContain("?secret=maple_gcp_") + + // Without an option an existing sink keeps its filter; with logs off there is none to carry. + const kept = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) + expect(kept.body.setup_script).toContain("LOG_FILTER_MODE='keep'") + const unknown = await harness.request("POST", `${path}/setup_scripts`, admin.secret, { + log_filter: "everything", + }) + expect(unknown.status).toBe(400) const patched = await harness.request("PATCH", path, admin.secret, { logs_enabled: false }) expect(patched.status).toBe(200) expect(patched.body).toEqual({ ...created.body, logs_enabled: false }) const metricsOnly = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) - expect(metricsOnly.body.setup_script).not.toContain("maple_gcp_") + expect(metricsOnly.body.setup_script).not.toContain("LOG_FILTER") expect(metricsOnly.body.setup_script).toContain("gcloud logging sinks delete") await harness.dispose() }) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.ts b/apps/api/src/routes/v2/integrations-gcp.http.ts index b029ce1b5d..1ca6f0c641 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.ts @@ -31,6 +31,9 @@ const toV2Connector = (connector: GcpConnector): V2GcpConnector => ({ created_at: isoTimestamp(connector.createdAt), last_log_received_at: isoTimestampOrNull(connector.lastLogReceivedAt), last_log_error: connector.lastLogError, + applied_logs_enabled: connector.appliedLogsEnabled, + applied_metrics_enabled: connector.appliedMetricsEnabled, + setup_reported_at: isoTimestampOrNull(connector.setupReportedAt), }) const auditMetadata = (connector: GcpConnector) => ({ @@ -96,7 +99,7 @@ export const HttpV2GcpIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "gcpIn const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly) const scripts = yield* gcp.scripts(tenant.orgId, params.id, { - excludeGkeContainerLogs: payload.exclude_gke_container_logs === true, + logFilter: payload.log_filter ?? "keep", }) return { object: "gcp_connector.setup_scripts" as const, diff --git a/docs/api-v2.md b/docs/api-v2.md index 88f3a214e8..0a1579affd 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -25,7 +25,7 @@ Within that group the role requirement is not uniform. Install, settings, unlink PlanetScale was the first promotion under that rule, and it shows what "a public dependency" means in practice: infrastructure-as-code setups drive `POST /v2/integrations/planetscale/metrics_token`, and a scripted caller needs a scoped API key, which only v2 has. That is the one step the OAuth flow cannot cover, because PlanetScale's metrics endpoints authenticate with service tokens instead of OAuth bearers. Once one operation of a provider is public the rest follows: splitting one provider across two tiers costs more than it buys, so the whole surface moved (status, connect, organization binding, disconnect, inventory, webhook config, query insights, events). The role split matches the chat group's: `status`, `databases`, `query_insights`, and `events` are ungated because the dashboard and service map render them for every member; everything that writes, plus `organizations` and `webhook_config` (it carries the signing secret), requires org-admin. -Google Cloud started on v2 for the same reason: there is no OAuth, so the whole setup is scriptable. A caller creates a connector for a project, a folder or an organization with a scoped key, chooses whether it forwards logs and whether Maple may read metrics and resources, and runs the returned `gcloud` script. `status` is ungated; creating, updating and deleting connectors and `setup_scripts` (it carries the push secret, and is a POST so a read-scoped key cannot fetch it) require org-admin. +Google Cloud started on v2 for the same reason: there is no OAuth, so the whole setup is scriptable. A caller creates a connector for a project, a folder or an organization with a scoped key, chooses whether it forwards logs and whether Maple may read metrics and resources, and runs the returned `gcloud` script. The script tells Maple what it applied, which `status` returns as `applied_logs_enabled` and `applied_metrics_enabled`, so a caller can see within seconds that the run finished. `status` is ungated; creating, updating and deleting connectors and `setup_scripts` (it carries the push secret, and is a POST so a read-scoped key cannot fetch it) require org-admin. Cloudflare, GitHub, Hazel, and Railway live on the internal `integrations` group (`/internal/integrations`) with no v2 counterpart. They predate v2 and nothing public depends on them. Per the rule above, each gets promoted individually if something does. Scope families are derived mechanically from the first path segment under `/v2`, so every provider mounted at `/v2/integrations/` shares the `integrations:read` / `integrations:write` family. diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts index 2da7128267..a1dd5610d1 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.test.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -68,7 +68,7 @@ const organizationScope: CreateGcpConnectorInput = { metricsEnabled: true, } -const scriptOptions = { excludeGkeContainerLogs: false } +const scriptOptions = { logFilter: "keep" } as const interface StoredConnector { readonly secret_ciphertext: string @@ -86,6 +86,8 @@ describe("GcpConnectorService", () => { assert.strictEqual(connector.projectId, "acme-prod") assert.isNull(connector.lastLogReceivedAt) assert.isNull(connector.lastLogError) + assert.isNull(connector.appliedLogsEnabled) + assert.isNull(connector.setupReportedAt) const { setupScript } = yield* gcp.scripts(orgId, connector.id, scriptOptions) const endpoint = /PUSH_ENDPOINT='([^']+)'/.exec(setupScript)?.[1] @@ -158,17 +160,21 @@ describe("GcpConnectorService", () => { const gcp = yield* GcpConnectorService assert.isTrue((yield* gcp.status(orgId)).metricsAvailable) const connector = yield* gcp.create(orgId, userId, organizationScope) - const both = yield* gcp.scripts(orgId, connector.id, { excludeGkeContainerLogs: true }) + const both = yield* gcp.scripts(orgId, connector.id, { + logFilter: "exclude_gke_container_logs", + }) assert.include(both.setupScript, `MAPLE_SERVICE_ACCOUNT='${MAPLE_ACCOUNT}'`) assert.include(both.setupScript, 'NOT resource.type="k8s_container"') assert.include(both.setupScript, '--organization="$SCOPE_ID" --include-children') assert.include(both.setupScript, "PROJECT_ID='acme-host'") - assert.notInclude(both.cleanupScript, "maple_gcp_") + assert.include(both.setupScript, "MAPLE_URL='http://127.0.0.1:3471/integrations?integration=gcp'") + // Both scripts tell Maple that they ran, so both carry the secret. + assert.include(both.cleanupScript, "?secret=maple_gcp_") const metricsOnly = yield* gcp.update(orgId, connector.id, { logsEnabled: false }) assert.deepStrictEqual(metricsOnly, { ...connector, logsEnabled: false }) const afterOptOut = yield* gcp.scripts(orgId, connector.id, scriptOptions) - assert.notInclude(afterOptOut.setupScript, "maple_gcp_") + assert.notInclude(afterOptOut.setupScript, "LOG_FILTER") assert.include(afterOptOut.setupScript, "gcloud logging sinks delete") const logsOnly = yield* gcp.update(orgId, connector.id, { @@ -188,10 +194,17 @@ describe("GcpConnectorService", () => { yield* Effect.promise(() => executeSql( testDb, - "UPDATE gcp_connectors SET last_received_at = $1, last_error = $2 WHERE id = $3", + `UPDATE gcp_connectors SET last_received_at = $1, last_error = $2, + applied_logs_enabled = true, setup_reported_at = $1 WHERE id = $3`, ["2026-10-08T09:12:00.000Z", "payload was not a LogEntry", connector.id], ), ) + // What a setup run reported describes Google Cloud, so no switch changes it. + const reported = { + ...connector, + appliedLogsEnabled: true, + setupReportedAt: Date.parse("2026-10-08T09:12:00.000Z"), + } const lastPush = { lastLogReceivedAt: Date.parse("2026-10-08T09:12:00.000Z"), lastLogError: "payload was not a LogEntry", @@ -199,13 +212,13 @@ describe("GcpConnectorService", () => { // Kept while logs stay on, and while they are off. const stillOn = yield* gcp.update(orgId, connector.id, { logsEnabled: true }) - assert.deepStrictEqual(stillOn, { ...connector, ...lastPush }) + assert.deepStrictEqual(stillOn, { ...reported, ...lastPush }) const off = yield* gcp.update(orgId, connector.id, { logsEnabled: false }) - assert.deepStrictEqual(off, { ...connector, ...lastPush, logsEnabled: false }) + assert.deepStrictEqual(off, { ...reported, ...lastPush, logsEnabled: false }) const backOn = yield* gcp.update(orgId, connector.id, { logsEnabled: true }) - assert.deepStrictEqual(backOn, connector) - assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, [connector]) + assert.deepStrictEqual(backOn, reported) + assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, [reported]) }).pipe(Effect.provide(makeLayer(testDb, MAPLE_ACCOUNT))) }) @@ -269,6 +282,7 @@ describe("GcpConnectorService", () => { const deleted = yield* gcp.delete(orgId, connector.id) assert.deepStrictEqual(deleted.connector, connector) assert.include(deleted.cleanupScript, gcpConnectorResourceNames(connector.id).sink) + assert.notInclude(deleted.cleanupScript, "maple_gcp_") assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, []) const again = yield* Effect.flip(gcp.delete(orgId, connector.id)) assert.strictEqual(again._tag, "@maple/http/errors/IntegrationsNotFoundError") diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index 6cf218c01e..8b8d8d9d8c 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -9,6 +9,7 @@ import { } from "@maple/domain/http" import { GcpConnectorId, + type GcpLogFilter, type GcpProjectId, type GcpResourceNumber, type GcpScopeType, @@ -38,6 +39,10 @@ export interface GcpConnector { /** Last log push the ingest gateway accepted from this connector. Null until the first one. */ readonly lastLogReceivedAt: number | null readonly lastLogError: string | null + /** What the latest setup script run reported it set up in Google Cloud. Null until one reports. */ + readonly appliedLogsEnabled: boolean | null + readonly appliedMetricsEnabled: boolean | null + readonly setupReportedAt: number | null } export type CreateGcpConnectorInput = Pick< @@ -76,7 +81,7 @@ export interface GcpConnectorServiceApi { readonly scripts: ( orgId: OrgId, connectorId: GcpConnectorId, - options: { readonly excludeGkeContainerLogs: boolean }, + options: { readonly logFilter: GcpLogFilter }, ) => Effect.Effect< { readonly setupScript: string; readonly cleanupScript: string }, IntegrationsNotFoundError | IntegrationsPersistenceError @@ -109,6 +114,9 @@ const toConnector = (row: GcpConnectorRow): GcpConnector => ({ createdAt: dateToMs(row.createdAt), lastLogReceivedAt: dateToMs(row.lastReceivedAt), lastLogError: row.lastError, + appliedLogsEnabled: row.appliedLogsEnabled, + appliedMetricsEnabled: row.appliedMetricsEnabled, + setupReportedAt: dateToMs(row.setupReportedAt), }) const notFound = () => new IntegrationsNotFoundError({ message: "No such Google Cloud connector." }) @@ -133,6 +141,7 @@ export class GcpConnectorService extends Context.Service ({ + connectorId, + scopeType, + scopeId: scopeType === "project" ? hostProject : resourceNumber, + projectId: hostProject, + mapleUrl: MAPLE_URL, +}) /** Each scope with the sink flag and IAM command the script must use for it. */ const SCOPES = { project: { - target: { connectorId, scopeType: "project", scopeId: hostProject, projectId: hostProject }, + target: target("project"), sink: "--project=acme-host", sinkWrite: "--project=acme-host", iam: "projects", id: "acme-host", + open: [], + found: [], }, folder: { - target: { connectorId, scopeType: "folder", scopeId: resourceNumber, projectId: hostProject }, + target: target("folder"), sink: "--folder=123456789012", sinkWrite: "--folder=123456789012 --include-children", iam: "resource-manager folders", id: "123456789012", + open: ["resource-manager folders describe 123456789012 --format=value(displayName)"], + found: [" ✓ Folder 123456789012 found (acme.com)"], }, organization: { - target: { connectorId, scopeType: "organization", scopeId: resourceNumber, projectId: hostProject }, + target: target("organization"), sink: "--organization=123456789012", sinkWrite: "--organization=123456789012 --include-children", iam: "organizations", id: "123456789012", + open: ["organizations describe 123456789012 --format=value(displayName)"], + found: [" ✓ Organization 123456789012 found (acme.com)"], }, -} as const satisfies Record> +} as const +type Scope = (typeof SCOPES)[keyof typeof SCOPES] const scopeTypes = ["project", "folder", "organization"] as const const setupInput = ( @@ -59,16 +75,24 @@ const setupInput = ( mapleServiceAccountEmail: MAPLE_ACCOUNT, logsEnabled: capabilities.logs, metricsEnabled: capabilities.metrics, - excludeGkeContainerLogs: false, + logFilter: "keep", }) +const setup = (input: GcpSetupScriptInput) => Effect.runSync(renderGcpSetupScript(input)) +const cleanup = (scope: Scope, pushEndpoint?: string) => + Effect.runSync(renderGcpCleanupScript({ ...scope.target, pushEndpoint })) + // A `gcloud` that records its commands, and each argument on its own line so a value split by -// bad quoting shows. `describe` answers as the test says: found, missing, or denied, and so does -// the service account list (a missing account is an empty list). The sink's writer identity is -// the one value a script reads back. +// bad quoting shows. It talks on stderr like the real one, which a script must not pass on. +// `describe` answers as the test says: found, missing, or denied, and so does the service account +// list (a missing account is an empty list). `GCLOUD_FAIL` refuses the commands that contain it. const FAKE_GCLOUD = `#!/usr/bin/env bash echo "$*" >> "$GCLOUD_LOG" printf "%s\\n" "$@" >> "$GCLOUD_LOG.args" +denied() { + echo "ERROR: (gcloud) PERMISSION_DENIED: Permission denied on resource (or it may not exist)." >&2 + exit 1 +} # For a disabled API gcloud asks whether to enable it, unless prompts are off. api_off() { if [ "$CLOUDSDK_CORE_DISABLE_PROMPTS" != 1 ]; then @@ -78,26 +102,35 @@ api_off() { echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2 exit 1 } +if [ -n "$GCLOUD_FAIL" ] && [[ "$*" == *"$GCLOUD_FAIL"* ]]; then denied; fi case "$*" in + "config get-value account") echo "$GCLOUD_ACCOUNT" ;; + "auth print-access-token") echo "ya29.token" ;; + "projects describe"*) echo "Acme Production" ;; + "organizations describe"* | "resource-manager folders describe"*) echo "acme.com" ;; + "billing projects describe"*) echo "$GCLOUD_BILLING" ;; *"sinks describe"*"--format"*) echo "${WRITER}" ;; *"service-accounts list"*) case "$GCLOUD_DESCRIBE" in found) echo "${ACCOUNT}" ;; missing) echo "Listed 0 items." >&2 ;; - denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; + denied) denied ;; api_off) api_off ;; esac ;; *" describe "*) case "$GCLOUD_DESCRIBE" in missing) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; - denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; + denied) denied ;; api_off) api_off ;; esac ;; *"remove-iam-policy-binding"*) case "$GCLOUD_UNBIND" in absent) echo "ERROR: Policy binding with the specified principal, role, and condition not found!" >&2; exit 1 ;; - denied) echo "ERROR: (gcloud) PERMISSION_DENIED: caller lacks permission" >&2; exit 1 ;; + denied) denied ;; esac ;; + *"sinks create"* | *"sinks update"*) + echo "Please remember to grant the writer the Pub/Sub Publisher role on the topic." >&2 ;; + *) echo "Operation \\"operations/acat.p2-1\\" finished successfully." >&2 ;; esac ` @@ -105,6 +138,11 @@ esac const FAKE_GCLOUD_ACCOUNT_DELETED = `#!/usr/bin/env bash echo "$*" >> "$GCLOUD_LOG" case "$*" in + "config get-value account") echo jane@acme.com ;; + "auth print-access-token") echo ya29.token ;; + "projects describe"*) echo "Acme Production" ;; + "resource-manager folders describe"*) echo "acme.com" ;; + "billing projects describe"*) echo True ;; *"sinks describe"*"--format"*) echo "${WRITER}" ;; *"service-accounts describe"*) echo "ERROR: (gcloud) PERMISSION_DENIED: Permission 'iam.serviceAccounts.get' denied on resource (or it may not exist)." >&2; exit 1 ;; *"service-accounts list"*) echo "Listed 0 items." >&2 ;; @@ -112,27 +150,63 @@ case "$*" in esac ` +// A `curl` for the two calls a script makes. The permission test answers with the permissions it +// was asked about, less \`CURL_LACKING\`; the report to Maple is recorded with its body. +const FAKE_CURL = `#!/usr/bin/env bash +url="\${!#}" +while [ $# -gt 0 ]; do + case "$1" in + --data) data="$2" ;; + @*) header="$(cat "\${1#@}")" ;; + esac + shift +done +case "$url" in + *testIamPermissions) + echo "$url $header" >> "$CURL_LOG" + [ "$CURL_IAM" = ok ] || exit 22 + lacking="\\"$CURL_LACKING\\"" + echo "\${data//$lacking/}" ;; + *) + echo "$url $data" >> "$CURL_LOG" + [ "$CURL_REPORT" = ok ] || exit 22 ;; +esac +` + const tempDirs: string[] = [] afterEach(() => { for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }) }) -/** Run a rendered script in bash against the fake gcloud; returns the commands it issued. */ -const run = ( - script: string, - options: { - describe?: "found" | "missing" | "denied" | "api_off" - unbind?: "ok" | "absent" | "denied" - gcloud?: string - } = {}, -) => { +interface RunOptions { + describe?: "found" | "missing" | "denied" | "api_off" + unbind?: "ok" | "absent" | "denied" + gcloud?: string + /** A substring of the gcloud commands to refuse. */ + fail?: string + account?: string + billing?: "True" | "False" + /** A permission the account does not hold, or `unanswered` when Google cannot be asked. */ + lacking?: string + report?: "ok" | "unreachable" + /** Extra arguments for the shell the text is fed to, and text typed after it. */ + shell?: ReadonlyArray + then?: string +} + +/** Feed a rendered script to bash against the fakes; returns what it printed and issued. */ +const run = (script: string, options: RunOptions = {}) => { const dir = mkdtempSync(join(tmpdir(), "maple-gcp-script-")) tempDirs.push(dir) - writeFileSync(join(dir, "gcloud"), options.gcloud ?? FAKE_GCLOUD) - chmodSync(join(dir, "gcloud"), 0o755) + const fakes = { gcloud: options.gcloud ?? FAKE_GCLOUD, curl: FAKE_CURL, sleep: "#!/usr/bin/env bash\n" } + for (const [name, source] of Object.entries(fakes)) { + writeFileSync(join(dir, name), source) + chmodSync(join(dir, name), 0o755) + } + const read = (file: string) => (existsSync(file) ? readFileSync(file, "utf8").trim().split("\n") : []) const log = join(dir, "gcloud.log") - const result = spawnSync("bash", [], { - input: script, + const result = spawnSync("bash", options.shell ?? [], { + input: script + (options.then ?? ""), cwd: dir, encoding: "utf8", env: { @@ -141,6 +215,13 @@ const run = ( GCLOUD_LOG: log, GCLOUD_DESCRIBE: options.describe ?? "missing", GCLOUD_UNBIND: options.unbind ?? "ok", + GCLOUD_FAIL: options.fail ?? "", + GCLOUD_ACCOUNT: options.account ?? "jane@acme.com", + GCLOUD_BILLING: options.billing ?? "True", + CURL_LOG: join(dir, "curl.log"), + CURL_IAM: options.lacking === "unanswered" ? "unanswered" : "ok", + CURL_LACKING: options.lacking ?? "", + CURL_REPORT: options.report ?? "ok", }, }) return { @@ -148,12 +229,32 @@ const run = ( status: result.status, stdout: result.stdout, stderr: result.stderr, - commands: existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : [], - args: existsSync(`${log}.args`) ? readFileSync(`${log}.args`, "utf8").split("\n") : [], + /** Every gcloud command but the access checks, which each test of them names itself. */ + commands: read(log).filter((command) => !isAccessCheck(command)), + access: read(log).filter(isAccessCheck), + args: read(`${log}.args`), + curl: read(join(dir, "curl.log")), } } -const logsSetupCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ +const isAccessCheck = (command: string) => + /^(config get-value|auth print-access-token|projects describe|organizations describe|resource-manager folders describe|billing projects describe)/.test( + command, + ) + +const REPORT = (members: string) => + new RegExp( + `^\\{"logName":"projects/acme-host/logs/maple-setup","timestamp":"\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\dZ","jsonPayload":\\{${members}\\}\\}$`, + ) + +const publish = (members: string) => + expect.stringMatching( + new RegExp( + `^pubsub topics publish ${NAME} --project=acme-host --message=${REPORT(members).source.slice(1)}`, + ), + ) + +const logsSetupCommands = (scope: Scope) => [ "services enable pubsub.googleapis.com logging.googleapis.com --project=acme-host", `pubsub topics describe ${NAME} --project=acme-host`, `pubsub topics create ${NAME} --project=acme-host`, @@ -164,9 +265,10 @@ const logsSetupCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ `logging sinks describe ${NAME} ${scope.sink} --format=value(writerIdentity)`, `projects add-iam-policy-binding acme-host --member=${WRITER} --role=roles/logging.logWriter --condition=None`, `pubsub topics add-iam-policy-binding ${NAME} --project=acme-host --member=${WRITER} --role=roles/pubsub.publisher`, + publish('"logs":true'), ] -const metricsSetupCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ +const metricsSetupCommands = (scope: Scope) => [ "services enable monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com iamcredentials.googleapis.com --project=acme-host", `iam service-accounts describe ${ACCOUNT} --project=acme-host`, `iam service-accounts create ${NAME} --project=acme-host --display-name=Maple metrics and resource reader`, @@ -176,7 +278,7 @@ const metricsSetupCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ `iam service-accounts add-iam-policy-binding ${ACCOUNT} --project=acme-host --member=serviceAccount:${MAPLE_ACCOUNT} --role=roles/iam.serviceAccountTokenCreator --condition=None`, ] -const logsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ +const logsRemovalCommands = (scope: Scope) => [ `logging sinks describe ${NAME} ${scope.sink}`, `logging sinks delete ${NAME} ${scope.sink} --quiet`, `pubsub subscriptions describe ${NAME} --project=acme-host`, @@ -190,7 +292,7 @@ const ACCOUNT_LOOKUP = `iam service-accounts list --project=acme-host --filter=e /** A command that only looks: a describe, or the service account list. */ const isLookup = (command: string) => command.includes(" describe ") || command === ACCOUNT_LOOKUP -const metricsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => [ +const metricsRemovalCommands = (scope: Scope) => [ ACCOUNT_LOOKUP, `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, @@ -198,75 +300,452 @@ const metricsRemovalCommands = (scope: (typeof SCOPES)[keyof typeof SCOPES]) => `iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`, ] +const UNFINISHED = "\nSetup did not finish. Nothing needs undoing: fix this and paste the script again.\n" + describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { const scope = SCOPES[scopeType] + const host = scopeType === "project" ? "Project" : "Host project" - it("sets up logs and metrics from nothing, at the right scope", () => { - const { status, stdout, commands } = run( - renderGcpSetupScript(setupInput(scopeType, { logs: true, metrics: true })), + it("sets up logs and metrics from nothing, at the right scope, one line per step", () => { + const { status, stdout, stderr, commands, access } = run( + setup(setupInput(scopeType, { logs: true, metrics: true })), ) expect(status).toBe(0) - expect(stdout).toContain("Maple setup complete.") - expect(commands).toEqual([...logsSetupCommands(scope), ...metricsSetupCommands(scope)]) + expect(stderr).toBe("") + expect(commands).toEqual([ + ...logsSetupCommands(scope), + ...metricsSetupCommands(scope), + publish('"metrics":true'), + ]) + expect(access).toEqual([ + "config get-value account", + "auth print-access-token", + ...scope.open, + "projects describe acme-host --format=value(name)", + "billing projects describe acme-host --format=value(billingEnabled)", + ]) + expect(stdout).toBe(`Maple setup for ${scopeType} ${scope.id} + Log forwarding on + Metrics and resources on + +Checking access + ✓ Signed in as jane@acme.com +${[...scope.found, ` ✓ ${host} acme-host found (Acme Production)`].join("\n")} + ✓ Billing is enabled + ✓ jane@acme.com has the permissions this script needs + +Log forwarding + ✓ APIs enabled (Pub/Sub, Cloud Logging) + ✓ Topic created + ✓ Push subscription created + ✓ Log sink created + ✓ Sink allowed to write to the project + ✓ Sink allowed to publish to the topic + ✓ Check message sent to Maple through the topic + +Metrics and resources + ✓ APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials) + ✓ Read-only service account created + ✓ Read-only roles granted (Monitoring Viewer, Cloud Asset Viewer, Service Usage Consumer) + ✓ Maple allowed to read as that account + ✓ Maple notified + +Done. Google Cloud is set up for Maple. + Maple confirms it within a minute: ${MAPLE_URL} + Logs: a new sink takes up to 10 minutes to forward its first entries. + Metrics: the first read lands within about 10 minutes. +`) }) it("with logs only, sets up logs and removes an earlier metrics setup", () => { - const script = renderGcpSetupScript(setupInput(scopeType, { logs: true, metrics: false })) + const script = setup(setupInput(scopeType, { logs: true, metrics: false })) expect(script).not.toContain(MAPLE_ACCOUNT) - const { status, commands } = run(script, { describe: "found" }) + const { status, stdout, commands, access } = run(script, { describe: "found" }) expect(status).toBe(0) - // Everything exists already, so the log pipeline is updated in place. + // Everything exists already, so the log pipeline is updated in place, filter untouched. expect(commands.filter((command) => command.includes(" create "))).toEqual([]) expect(commands).toContain( - `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(false)} ${scope.sinkWrite}`, + `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} ${scope.sinkWrite}`, ) expect(commands).toContainEqual( expect.stringMatching(/^pubsub subscriptions update .*--push-no-wrapper/), ) - expect(commands.slice(-metricsRemovalCommands(scope).length)).toEqual(metricsRemovalCommands(scope)) + expect(commands.slice(-metricsRemovalCommands(scope).length - 1)).toEqual([ + ...metricsRemovalCommands(scope), + publish('"metrics":false'), + ]) + // Billing only matters to Cloud Monitoring. + expect(access).not.toContainEqual(expect.stringContaining("billing")) + expect(stdout).toContain(` +Metrics and resources + ✓ Read-only roles removed + ✓ Read-only service account deleted + ✓ Maple notified + +Done. Google Cloud matches your Maple switches. + Maple confirms it within a minute: ${MAPLE_URL} +`) }) - it("with metrics only, removes an earlier log pipeline and carries no secret", () => { - const script = renderGcpSetupScript(setupInput(scopeType, { logs: false, metrics: true })) - expect(script).not.toContain(SECRET) + it("with metrics only, removes an earlier log pipeline and reports without a topic", () => { + const script = setup(setupInput(scopeType, { logs: false, metrics: true })) expect(script).not.toContain("LOG_FILTER") - const { status, commands } = run(script, { describe: "found" }) + const { status, stdout, commands, curl } = run(script, { describe: "found" }) expect(status).toBe(0) expect(commands.slice(0, logsRemovalCommands(scope).length)).toEqual(logsRemovalCommands(scope)) expect(commands).not.toContainEqual(expect.stringContaining("pubsub.googleapis.com logging")) + expect(commands).not.toContainEqual(expect.stringContaining("topics publish")) expect(commands).toContain( `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, ) + expect(curl.filter((call) => call.startsWith(PUSH_ENDPOINT))).toEqual([ + expect.stringMatching(REPORT('"logs":false').source.slice(1)), + expect.stringMatching(REPORT('"metrics":true').source.slice(1)), + ]) + expect(stdout).toContain(` Log forwarding off, removing + Metrics and resources on +`) + expect(stdout).toContain(` +Log forwarding + ✓ Log sink deleted + ✓ Push subscription deleted + ✓ Topic deleted + ✓ Maple notified +`) }) - it("cleans up everything, and touches nothing when nothing exists", () => { - const script = renderGcpCleanupScript(scope.target) - expect(script).not.toContain("secret") + it("cleans up everything and tells Maple, and touches nothing when nothing exists", () => { + const script = cleanup(scope, PUSH_ENDPOINT) const existing = run(script, { describe: "found" }) expect(existing.status).toBe(0) expect(existing.commands).toEqual([...logsRemovalCommands(scope), ...metricsRemovalCommands(scope)]) + expect(existing.curl).toEqual([ + expect.stringMatching(REPORT('"logs":false,"metrics":false').source.slice(1)), + ]) + expect(existing.stdout).toBe(`Maple cleanup for ${scopeType} ${scope.id} + +Checking access + ✓ Signed in as jane@acme.com +${[...scope.found, ` ✓ ${host} acme-host found (Acme Production)`].join("\n")} + +Log forwarding + ✓ Log sink deleted + ✓ Push subscription deleted + ✓ Topic deleted + +Metrics and resources + ✓ Read-only roles removed + ✓ Read-only service account deleted + +Done. Everything the setup script created is gone. + Left in place: the APIs it switched on, and the Logs Writer role of Google's logging + service account on acme-host, which other sinks share. + Now disconnect in Maple: ${MAPLE_URL} +`) const fresh = run(script) expect(fresh.status).toBe(0) expect(fresh.commands.every(isLookup)).toBe(true) + expect(fresh.stdout.match(/ {2}✓ Nothing left to remove/g)).toHaveLength(2) + }) +}) + +describe("the text that is copied", () => { + const script = setup(setupInput("project", { logs: true, metrics: true })) + + it("is the script as a here-document for a child bash", () => { + const lines = script.split("\n") + expect(lines.slice(0, 3)).toEqual([ + ` { [ -z "\${BASH_VERSION-}" ] || case "$(history 1)" in *MAPLE_SETUP_SCRIPT*) history -d -1 ;; esac 2>/dev/null`, + "bash /dev/fd/3 3<<'MAPLE_SETUP_SCRIPT'", + "#!/usr/bin/env bash", + ]) + expect(script.endsWith("\nMAPLE_SETUP_SCRIPT\n}\n")).toBe(true) + // An interactive zsh runs a comment as a command. + expect( + lines + .slice(0, 2) + .concat(lines.slice(-3)) + .some((line) => line.startsWith("#")), + ).toBe(false) + // Plain ASCII survives a paste under any locale; the marks it prints are spelled as bytes. + expect([...script].every((character) => character.charCodeAt(0) < 128)).toBe(true) + + const cleanupLines = cleanup(SCOPES.project).split("\n") + expect(cleanupLines[1]).toBe("bash /dev/fd/3 3<<'MAPLE_CLEANUP_SCRIPT'") + expect(cleanupLines.slice(-3)).toEqual(["MAPLE_CLEANUP_SCRIPT", "}", ""]) + }) + + it("leaves an interactive shell alive, with its options and the script's exit status", () => { + const { stdout } = run(script, { + fail: "sinks create", + shell: ["--norc", "--noprofile", "-i"], + then: "echo STATUS=$? ALIVE; set -o | grep -E 'errexit|nounset'; history | grep -c 'MAPLE_SETUP_SCRIP[T]'\n", + }) + expect(stdout).toContain(UNFINISHED) + expect(stdout).toContain("STATUS=1 ALIVE") + expect(stdout).toMatch(/errexit\s+off\nnounset\s+off/) + // bash 5 can name the running command's history entry; older ones keep the paste. + const bashMajor = Number( + spawnSync("bash", ["-c", "echo $BASH_VERSINFO"], { encoding: "utf8" }).stdout, + ) + if (bashMajor >= 5) expect(stdout.trim().endsWith("\n0")).toBe(true) + }) + + it("is refused when a value would end the here-document early", () => { + const hostile = { ...setupInput("project", { logs: true, metrics: true }) } + const result = Effect.runSync( + Effect.result( + renderGcpSetupScript({ + ...hostile, + mapleServiceAccountEmail: "x\nMAPLE_SETUP_SCRIPT\ntouch pwned", + }), + ), + ) + expect(result._tag).toBe("Failure") }) }) describe("renderGcpSetupScript", () => { + it("reads a re-run as already in place and keeps the sink's filter", () => { + const { status, stdout, commands } = run( + setup(setupInput("project", { logs: true, metrics: true })), + { + describe: "found", + }, + ) + expect(status).toBe(0) + expect(commands.filter((command) => command.includes(" create "))).toEqual([]) + expect(commands).not.toContainEqual(expect.stringContaining("--log-filter")) + expect(stdout).toContain(` +Log forwarding + ✓ APIs enabled (Pub/Sub, Cloud Logging) + ✓ Topic already exists + ✓ Push subscription up to date + ✓ Log sink up to date (filter kept) + ✓ Sink allowed to write to the project + ✓ Sink allowed to publish to the topic + ✓ Check message sent to Maple through the topic + +Metrics and resources + ✓ APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials) + ✓ Read-only service account already exists + ✓ Read-only roles granted (Monitoring Viewer, Cloud Asset Viewer, Service Usage Consumer) + ✓ Maple allowed to read as that account + ✓ Maple notified + +Done. Everything was already in place. + Maple confirms it within a minute: ${MAPLE_URL} +`) + expect(stdout).not.toMatch(/Logs: {4}|Metrics: /) + }) + + it("replaces the sink's filter only when asked to", () => { + const filtered = (logFilter: GcpSetupScriptInput["logFilter"]) => + setup({ ...setupInput("project", { logs: true, metrics: false }), logFilter }) + expect(filtered("keep")).toContain(`LOG_FILTER_MODE='keep'\nLOG_FILTER='${gcpLogFilter(false)}'`) + expect(filtered("default")).toContain(`LOG_FILTER_MODE='set'\nLOG_FILTER='${gcpLogFilter(false)}'`) + expect(filtered("exclude_gke_container_logs")).toContain( + `LOG_FILTER_MODE='set'\nLOG_FILTER='${gcpLogFilter(true)}'`, + ) + + const { stdout, commands } = run(filtered("exclude_gke_container_logs"), { describe: "found" }) + expect(stdout).toContain(" ✓ Log sink up to date (filter replaced)") + expect(commands).toContain( + `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(true)} --project=acme-host`, + ) + }) + + it("shows Google's answer and what to do when a step is refused, and stops there", () => { + const { status, stdout, commands } = run( + setup(setupInput("project", { logs: true, metrics: true })), + { + fail: "sinks create", + }, + ) + expect(status).toBe(1) + expect(commands.at(-1)).toContain("logging sinks create") + expect( + stdout.endsWith(` +Log forwarding + ✓ APIs enabled (Pub/Sub, Cloud Logging) + ✓ Topic created + ✓ Push subscription created + ✗ Couldn't create the log sink. + + ERROR: (gcloud) PERMISSION_DENIED: Permission denied on resource (or it may not exist). + + What to do: jane@acme.com needs Logs Configuration Writer on project acme-host. +${UNFINISHED}`), + ).toBe(true) + }) + + it("names the role of the scope for a refused grant there", () => { + const { stdout } = run(setup(setupInput("organization", { logs: false, metrics: true })), { + fail: "organizations add-iam-policy-binding", + }) + expect(stdout).toContain(` ✗ Couldn't grant the read-only roles.`) + expect(stdout).toContain( + "What to do: jane@acme.com needs Organization Administrator on organization 123456789012.", + ) + }) + + it("changes nothing when the project cannot be opened", () => { + const { status, stdout, commands } = run( + setup(setupInput("project", { logs: true, metrics: true })), + { + fail: "projects describe", + }, + ) + expect(status).toBe(1) + expect(commands).toEqual([]) + expect(stdout).toContain(` +Checking access + ✓ Signed in as jane@acme.com + ✗ Can't open project acme-host as jane@acme.com. + + ERROR: (gcloud) PERMISSION_DENIED: Permission denied on resource (or it may not exist). + + What to do: Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). If it is right, this account has no access to the project. +${UNFINISHED}`) + }) + + it("needs the organization to open for metrics, and leaves that to the permissions for logs alone", () => { + const options = { fail: "organizations describe" } + const both = run(setup(setupInput("organization", { logs: true, metrics: true })), options) + expect(both.status).toBe(1) + expect(both.commands).toEqual([]) + expect(both.stdout).toContain(" ✗ Can't open organization 123456789012 as jane@acme.com.") + expect(both.stdout).toContain( + "What to do: Check the ID with: gcloud organizations list. If it is right, this account needs Organization Administrator there.", + ) + + // Logs Configuration Writer does not include reading the organization. + const logsOnly = run(setup(setupInput("organization", { logs: true, metrics: false })), options) + expect(logsOnly.status).toBe(0) + expect(logsOnly.stdout).not.toContain("Organization 123456789012 found") + }) + + it("stops before the first change without billing, a sign-in, or gcloud", () => { + const script = setup(setupInput("project", { logs: true, metrics: true })) + const unbilled = run(script, { billing: "False" }) + expect(unbilled.commands).toEqual([]) + expect(unbilled.stdout).toContain(` ✗ Project acme-host has no billing account. + + What to do: Cloud Monitoring only answers for projects with billing. Link one at https://console.cloud.google.com/billing/linkedaccount?project=acme-host +${UNFINISHED}`) + + const signedOut = run(script, { account: "" }) + expect(signedOut.commands).toEqual([]) + expect(signedOut.stdout).toContain(` ✗ You are not signed in to Google Cloud. + + What to do: In Cloud Shell, click Authorize when it asks. Elsewhere, run: gcloud auth login +`) + + // The shell the text is fed to still has to find bash. + const bash = spawnSync("bash", ["-c", "command -v bash"], { encoding: "utf8" }).stdout.trim() + const dir = mkdtempSync(join(tmpdir(), "maple-gcp-bin-")) + tempDirs.push(dir) + symlinkSync(bash, join(dir, "bash")) + const bare = spawnSync(bash, [], { input: script, encoding: "utf8", env: { PATH: dir } }) + expect(bare.status).toBe(1) + expect(bare.stdout).toContain(` ✗ gcloud is not installed here. + + What to do: Run this script in Cloud Shell: https://shell.cloud.google.com +`) + }) + + it("names a missing permission before the first change", () => { + const script = setup(setupInput("organization", { logs: true, metrics: true })) + const onHost = run(script, { lacking: "resourcemanager.projects.setIamPolicy" }) + expect(onHost.status).toBe(1) + expect(onHost.commands).toEqual([]) + expect(onHost.stdout) + .toContain(` ✗ jane@acme.com is missing permissions on project acme-host: resourcemanager.projects.setIamPolicy. + + What to do: Ask for Owner there, or have an administrator run this script. +`) + expect(onHost.curl).toEqual([ + "https://cloudresourcemanager.googleapis.com/v3/projects/acme-host:testIamPermissions Authorization: Bearer ya29.token", + ]) + + const onScope = run(script, { lacking: "logging.sinks.create" }) + expect(onScope.commands).toEqual([]) + expect(onScope.stdout) + .toContain(` ✗ jane@acme.com is missing permissions on organization 123456789012: logging.sinks.create. + + What to do: Ask for Logs Configuration Writer and Organization Administrator there, or have an administrator run this script. If the ID looks wrong, check it with: gcloud organizations list +`) + expect(onScope.curl.at(-1)).toContain("/v3/organizations/123456789012:testIamPermissions") + + // No answer is not a refusal: the steps report for themselves. + const unanswered = run(script, { lacking: "unanswered" }) + expect(unanswered.status).toBe(0) + expect(unanswered.stdout).not.toContain("has the permissions") + }) + + it("finishes when Maple cannot be reached, and does not claim a confirmation", () => { + const { status, stdout } = run(setup(setupInput("project", { logs: false, metrics: true })), { + report: "unreachable", + }) + expect(status).toBe(0) + expect(stdout).toContain( + " • Couldn't reach Maple to confirm. Maple shows the connection as pending until a later run reaches it.\n", + ) + expect(stdout).toContain("Done. Google Cloud is set up for Maple.") + expect(stdout).not.toContain("Maple confirms it") + + const unpublished = run(setup(setupInput("project", { logs: true, metrics: true })), { + fail: "topics publish", + }) + expect(unpublished.status).toBe(0) + expect(unpublished.stdout).not.toContain("Maple confirms it") + }) + + it("waits for a new service account only while Google says it does not exist", () => { + // The first two grants are answered as for an account IAM has not heard of yet. + const slow = `#!/usr/bin/env bash +echo "$*" >> "$GCLOUD_LOG" +case "$*" in + "config get-value account") echo jane@acme.com ;; + "auth print-access-token") echo ya29.token ;; + "projects describe"*) echo "Acme Production" ;; + "billing projects describe"*) echo True ;; + *" describe "*) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; + *"add-iam-policy-binding"*) + if [ "$(grep -c add-iam-policy-binding "$GCLOUD_LOG")" -le 2 ]; then + echo "ERROR: (gcloud) INVALID_ARGUMENT: Service account ${ACCOUNT} does not exist." >&2 + exit 1 + fi ;; +esac +` + const { status, stdout, commands } = run( + setup(setupInput("project", { logs: false, metrics: true })), + { gcloud: slow }, + ) + expect(status).toBe(0) + expect(stdout.match(/ {2}… waiting for Google to publish the new service account\n/g)).toHaveLength(1) + expect(commands.filter((command) => command.includes("roles/monitoring.viewer"))).toHaveLength(3) + }) + it("leaves an enabled metrics setup alone when the deployment has lost its Google identity", () => { - const script = renderGcpSetupScript({ + const script = setup({ ...setupInput("organization", { logs: true, metrics: true }), mapleServiceAccountEmail: undefined, }) expect(script).toContain("Metrics and resources: on, unavailable") - const { status, commands } = run(script, { describe: "found" }) + const { status, stdout, commands } = run(script, { describe: "found" }) expect(status).toBe(0) + expect(stdout).toContain( + " Metrics and resources not available on this Maple deployment, left as is\n", + ) + expect(stdout).not.toContain("\nMetrics and resources\n") expect(commands).not.toContainEqual(expect.stringContaining("service-accounts")) expect(commands).not.toContainEqual(expect.stringContaining("remove-iam-policy-binding")) }) it("still deletes the service account when one of its role bindings is already gone", () => { - const script = renderGcpCleanupScript(SCOPES.folder.target) + const script = cleanup(SCOPES.folder) const absent = run(script, { describe: "found", unbind: "absent" }) expect(absent.commands.at(-1)).toBe( `iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`, @@ -277,65 +756,80 @@ describe("renderGcpSetupScript", () => { const denied = run(script, { describe: "found", unbind: "denied" }) expect(denied.commands).not.toContainEqual(expect.stringContaining("service-accounts delete")) expect(denied.status).toBe(1) + expect(denied.stdout).toContain(` ✗ Couldn't remove the read-only roles.`) + expect(denied.stdout).toContain( + "What to do: jane@acme.com needs Folder IAM Admin on folder 123456789012.", + ) + expect(denied.stdout).toContain( + "\nCleanup did not finish. Fix this and paste the script again: it continues where it stopped.\n", + ) }) it("keeps the topic when it cannot tell whether the sink is gone", () => { - const sinkDenied = `#!/usr/bin/env bash -echo "$*" >> "$GCLOUD_LOG" -case "$*" in *"sinks describe"*) echo "ERROR: PERMISSION_DENIED" >&2; exit 1 ;; esac -` - const { status, commands } = run(renderGcpCleanupScript(SCOPES.organization.target), { - gcloud: sinkDenied, - }) + const { status, stdout, commands } = run(cleanup(SCOPES.organization), { fail: "sinks describe" }) expect(commands).not.toContainEqual(expect.stringContaining("pubsub")) - expect(commands).not.toContainEqual(expect.stringContaining("service-accounts delete")) + expect(commands).not.toContainEqual(expect.stringContaining("service-accounts")) expect(status).toBe(1) + expect(stdout).toContain(` ✗ Couldn't check whether the log sink still exists.`) + expect(stdout).toContain( + "What to do: jane@acme.com needs Logs Configuration Writer on organization 123456789012.", + ) }) it("does not take a resource it may not look at for one that is gone", () => { - const cleanup = run(renderGcpCleanupScript(SCOPES.organization.target), { describe: "denied" }) - expect(cleanup.commands.every(isLookup)).toBe(true) - expect(cleanup.stderr).toContain("PERMISSION_DENIED") - expect(cleanup.stderr).toContain("Not everything could be removed") - expect(cleanup.status).toBe(1) + const removal = run(cleanup(SCOPES.organization), { describe: "denied" }) + expect(removal.commands).toEqual([`logging sinks describe ${NAME} --organization=123456789012`]) + expect(removal.stdout).toContain("PERMISSION_DENIED") + expect(removal.status).toBe(1) // Setting up still works: a failed look falls through to the create, which reports for itself. - const setup = run(renderGcpSetupScript(setupInput("organization", { logs: true, metrics: true })), { + const created = run(setup(setupInput("organization", { logs: true, metrics: true })), { describe: "denied", }) - expect(setup.status).toBe(0) + expect(created.status).toBe(0) }) it("reads a disabled API as nothing to remove in the host project, but not for the sink", () => { - const { status, stderr, commands } = run(renderGcpCleanupScript(SCOPES.organization.target), { - describe: "api_off", - }) + const sinkUnknown = run(cleanup(SCOPES.organization), { describe: "api_off" }) + expect(sinkUnknown.commands).toEqual([`logging sinks describe ${NAME} --organization=123456789012`]) + expect(sinkUnknown.stdout).toContain("has not been used") + expect(sinkUnknown.status).toBe(1) + + // With the sink known to be gone, the host project's disabled APIs hold nothing. + const hostOff = `#!/usr/bin/env bash +echo "$*" >> "$GCLOUD_LOG" +case "$*" in + "config get-value account") echo jane@acme.com ;; + "auth print-access-token") echo ya29.token ;; + "projects describe"*) echo "Acme Production" ;; + *"sinks describe"*) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; + *) echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2; exit 1 ;; +esac +` + const { status, stdout, commands } = run(cleanup(SCOPES.project), { gcloud: hostOff }) + expect(status).toBe(0) expect(commands.every(isLookup)).toBe(true) - expect(stderr.match(/has not been used/g)).toHaveLength(1) - expect(status).toBe(1) + expect(stdout.match(/ {2}✓ Nothing left to remove/g)).toHaveLength(2) }) it("finishes when the service account was deleted by an earlier run", () => { - const cleanup = run(renderGcpCleanupScript(SCOPES.project.target), { - gcloud: FAKE_GCLOUD_ACCOUNT_DELETED, - }) - expect(cleanup.stdout).toContain("Maple cleanup complete.") - expect(cleanup.status).toBe(0) - expect(cleanup.commands.every(isLookup)).toBe(true) + const removal = run(cleanup(SCOPES.project), { gcloud: FAKE_GCLOUD_ACCOUNT_DELETED }) + expect(removal.stdout).toContain("Done. Everything the setup script created is gone.") + expect(removal.status).toBe(0) + expect(removal.commands.every(isLookup)).toBe(true) // So does a setup script with metrics off, run a second time. - const metricsOff = run(renderGcpSetupScript(setupInput("project", { logs: true, metrics: false })), { + const metricsOff = run(setup(setupInput("project", { logs: true, metrics: false })), { gcloud: FAKE_GCLOUD_ACCOUNT_DELETED, }) expect(metricsOff.status).toBe(0) - expect(metricsOff.commands.at(-1)).toBe(ACCOUNT_LOOKUP) + expect(metricsOff.commands.slice(-2)).toEqual([ACCOUNT_LOOKUP, publish('"metrics":false')]) }) it("creates the service account again after an earlier run deleted it", () => { - const { status, commands } = run( - renderGcpSetupScript(setupInput("folder", { logs: false, metrics: true })), - { gcloud: FAKE_GCLOUD_ACCOUNT_DELETED }, - ) + const { status, commands } = run(setup(setupInput("folder", { logs: false, metrics: true })), { + gcloud: FAKE_GCLOUD_ACCOUNT_DELETED, + }) expect(status).toBe(0) // The new account has none of the deleted one's roles: all four grants are made again. expect(commands.slice(-metricsSetupCommands(SCOPES.folder).length)).toEqual( @@ -343,8 +837,18 @@ case "$*" in *"sinks describe"*) echo "ERROR: PERMISSION_DENIED" >&2; exit 1 ;; ) }) + it("carries no secret and reports nothing once the connector is gone", () => { + const script = cleanup(SCOPES.project) + expect(script).not.toContain("secret") + expect(script).not.toContain("PUSH_ENDPOINT") + const { status, stdout, curl } = run(script, { describe: "found" }) + expect(status).toBe(0) + expect(curl).toEqual([]) + expect(stdout).not.toContain("disconnect in Maple") + }) + it("hands gcloud each value as one argument", () => { - const { args } = run(renderGcpSetupScript(setupInput("folder", { logs: true, metrics: true }))) + const { args } = run(setup(setupInput("folder", { logs: true, metrics: true }))) expect(args).toEqual( expect.arrayContaining([ `--log-filter=${gcpLogFilter(false)}`, @@ -352,6 +856,7 @@ case "$*" in *"sinks describe"*) echo "ERROR: PERMISSION_DENIED" >&2; exit 1 ;; "--display-name=Maple metrics and resource reader", "--format=value(writerIdentity)", `--member=${WRITER}`, + expect.stringMatching(`^--message=${REPORT('"logs":true').source.slice(1)}`), ]), ) }) @@ -361,20 +866,15 @@ case "$*" in *"sinks describe"*) echo "ERROR: PERMISSION_DENIED" >&2; exit 1 ;; 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC"', ) expect(gcpLogFilter(true)).toBe(`${gcpLogFilter(false)} AND NOT resource.type="k8s_container"`) - expect( - renderGcpSetupScript({ - ...setupInput("project", { logs: true, metrics: false }), - excludeGkeContainerLogs: true, - }), - ).toContain(`LOG_FILTER='${gcpLogFilter(true)}'`) }) it("passes interpolated values to gcloud verbatim, whatever they contain", () => { const hostile = `x'; touch pwned; echo '$(touch pwned)` const { dir, status, commands } = run( - renderGcpSetupScript({ + setup({ ...setupInput("project", { logs: false, metrics: true }), mapleServiceAccountEmail: hostile, + mapleUrl: hostile, }), ) expect(status).toBe(0) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 8e2d31d1c4..7fa6e7cb86 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -1,9 +1,19 @@ import { gcpConnectorResourceNames } from "@maple/domain/gcp" -import type { GcpConnectorId, GcpProjectId, GcpResourceNumber, GcpScopeType } from "@maple/domain/primitives" +import { IntegrationsPersistenceError } from "@maple/domain/http" +import type { + GcpConnectorId, + GcpLogFilter, + GcpProjectId, + GcpResourceNumber, + GcpScopeType, +} from "@maple/domain/primitives" +import { Effect } from "effect" /** Single-quote a value for bash. Every value that reaches a script goes through here. */ const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` +const DOCS_URL = "https://maple.dev/docs/integrations/gcp" + // Data Access audit logs record every API read, and load balancer health-check probes hit each // backend every few seconds: both are high volume and say little about the workload. const DEFAULT_LOG_FILTER = [ @@ -23,29 +33,30 @@ export interface GcpScriptTarget { readonly scopeId: GcpProjectId | GcpResourceNumber /** The host project: topic, subscription and service account are created here. */ readonly projectId: GcpProjectId + /** The Google Cloud integration page, where the script sends its reader back to. */ + readonly mapleUrl: string } -// How each scope is addressed: the flag that places a log sink, the command family that edits its -// IAM policy, and the roles the person running the script needs there. +// How each scope is addressed: the flag that places a log sink, the command family that reads it +// and edits its IAM policy, the role that may edit that policy, and how to look its ID up. const SCOPES = { project: { sink: '--project="$SCOPE_ID"', - iam: "gcloud projects", - needs: "# Owner on the project (PROJECT_ID below).", + gcloud: "gcloud projects", + iamRole: "Owner", + list: "gcloud projects list", }, folder: { sink: '--folder="$SCOPE_ID"', - iam: "gcloud resource-manager folders", - needs: `# Owner on the host project (PROJECT_ID below), which holds Maple's Pub/Sub topic, -# subscription and service account. On the folder (SCOPE_ID below): Logs Configuration -# Writer for the log sink, and Folder IAM Admin for the read-only roles.`, + gcloud: "gcloud resource-manager folders", + iamRole: "Folder IAM Admin", + list: "gcloud resource-manager folders list --organization=ORGANIZATION_ID", }, organization: { sink: '--organization="$SCOPE_ID"', - iam: "gcloud organizations", - needs: `# Owner on the host project (PROJECT_ID below), which holds Maple's Pub/Sub topic, -# subscription and service account. On the organization (SCOPE_ID below): Logs Configuration -# Writer for the log sink, and Organization Administrator for the read-only roles.`, + gcloud: "gcloud organizations", + iamRole: "Organization Administrator", + list: "gcloud organizations list", }, } as const @@ -53,164 +64,382 @@ const SCOPES = { const includeChildren = (scopeType: GcpScopeType): string => scopeType === "project" ? "" : " --include-children" -const variables = (target: GcpScriptTarget): string => { +const variables = (target: GcpScriptTarget, more: ReadonlyArray): string => { const names = gcpConnectorResourceNames(target.connectorId) - return `PROJECT_ID=${sh(target.projectId)} + return ` +# ---- This connection ---- +# PROJECT_ID is the host project: it holds Maple's Pub/Sub topic, subscription and service account. +PROJECT_ID=${sh(target.projectId)} SCOPE_ID=${sh(target.scopeId)} TOPIC=${sh(names.topic)} SUBSCRIPTION=${sh(names.subscription)} SINK=${sh(names.sink)} SERVICE_ACCOUNT=${sh(names.serviceAccountId)} -SERVICE_ACCOUNT_EMAIL="$SERVICE_ACCOUNT@$PROJECT_ID.iam.gserviceaccount.com"` +SERVICE_ACCOUNT_EMAIL="$SERVICE_ACCOUNT@$PROJECT_ID.iam.gserviceaccount.com" +${more.map((line) => `${line}\n`).join("")}MAPLE_URL=${sh(target.mapleUrl)} +SCOPE="${target.scopeType} $SCOPE_ID" +OWNER="Owner on project $PROJECT_ID" +SINK_ROLE="Logs Configuration Writer on $SCOPE" +IAM_ROLE="${SCOPES[target.scopeType].iamRole} on $SCOPE" +` } -const logsSetup = (scopeType: GcpScopeType, pushEndpoint: string): string => ` -# ---- Logs: on ---- -PUSH_ENDPOINT=${sh(pushEndpoint)} -DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" +// Google's own output is shown only for a step that fails, under what failed and above what to +// do about it. `stop` ends the script: every step can be run again, so nothing is rolled back. +const output = (unfinished: string) => ` +# ---- What this script prints ---- +# Its marks are spelled as bytes: the text you paste is plain ASCII, which every terminal and +# locale passes on unchanged. +section() { printf '\\n%s\\n' "$1"; } +ok() { printf ' \\342\\234\\223 %s\\n' "$1"; } + +# stop [Google's answer] +stop() { + printf ' \\342\\234\\227 %s\\n' "$1" + if [ -n "\${3-}" ]; then printf '\\n'; printf '%s\\n' "$3" | sed 's/^/ /'; fi + printf '\\n What to do: %s\\n' "$2" + printf '\\n%s\\n' ${sh(unfinished)} + exit 1 +} -# APIs in the host project: Pub/Sub carries the log entries, Cloud Logging routes them. -gcloud services enable pubsub.googleapis.com logging.googleapis.com --project="$PROJECT_ID" +# fix : what to do about a refused step. +fix() { + case "$1" in + *"requires billing"* | *BILLING_DISABLED*) + printf '%s' "Project $PROJECT_ID has no active billing account. Link one at https://console.cloud.google.com/billing/linkedaccount?project=$PROJECT_ID" ;; + *allowedPolicyMember* | *"permitted customer"*) + printf '%s' "An organization policy (domain restricted sharing) blocks this grant. Ask an Organization Policy Administrator to lift it for project $PROJECT_ID while this script runs. See ${DOCS_URL}#domain-restricted-sharing" ;; + *PERMISSION_DENIED* | *"does not have permission"* | *"denied on resource"*) + printf '%s' "$ACCOUNT needs $2." ;; + *) printf '%s' "Read Google's answer above. If it is unclear, send this output to support@maple.dev." ;; + esac +} -if ! gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID" >/dev/null 2>&1; then - gcloud pubsub topics create "$TOPIC" --project="$PROJECT_ID" -fi +# try +try() { + local failed="$1" role="$2" output + shift 2 + output="$("$@" 2>&1)" || stop "$failed" "$(fix "$output" "$role")" "$output" +} -# One LogEntry per request, unwrapped. A day of retention bounds what is replayed after an -# outage, and the subscription never expires while it is idle. -SUBSCRIPTION_FLAGS=( - --project="$PROJECT_ID" - --push-endpoint="$PUSH_ENDPOINT" --push-no-wrapper - --ack-deadline=30 --message-retention-duration=1d --expiration-period=never - --min-retry-delay=10s --max-retry-delay=600s -) -if gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID" >/dev/null 2>&1; then - gcloud pubsub subscriptions update "$SUBSCRIPTION" "\${SUBSCRIPTION_FLAGS[@]}" -else - gcloud pubsub subscriptions create "$SUBSCRIPTION" --topic="$TOPIC" "\${SUBSCRIPTION_FLAGS[@]}" -fi +# run +run() { + local worked="$1" + shift + try "$@" + ok "$worked" +} -if gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} >/dev/null 2>&1; then - gcloud logging sinks update "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} -else - gcloud logging sinks create "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} -fi +# report : the message that tells Maple what a run did. It has +# the form of a log entry; Maple reads it and does not store it. +report() { + printf '{"logName":"projects/%s/logs/maple-setup","timestamp":"%s","jsonPayload":{%s}}' \\ + "$PROJECT_ID" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$1" +} +` -# The sink writes as a Google-managed identity. Google requires Logs Writer for it on the project -# that holds the destination, and it needs Pub/Sub Publisher on this one topic. -WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} --format='value(writerIdentity)')" -gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ - --member="$WRITER_IDENTITY" --role=roles/logging.logWriter --condition=None >/dev/null -gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ - --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher >/dev/null +const POST_REPORT = `curl -fsS --max-time 15 -o /dev/null -X POST -H 'Content-Type: application/json'` + +// Through the topic while there is one: the message arriving proves topic, subscription and +// Maple are wired up. Maple not hearing of a run must never fail it. +const notify = (throughTopic: boolean) => ` +# notify +CONFIRMED=1 +notify() { + if ${ + throughTopic + ? `gcloud pubsub topics publish "$TOPIC" --project="$PROJECT_ID" --message="$(report "$2")"` + : `${POST_REPORT} --data "$(report "$2")" "$PUSH_ENDPOINT"` + } >/dev/null 2>&1; then + ok "$1" + else + printf " \\342\\200\\242 Couldn't reach Maple to confirm. Maple shows the connection as pending until a later run reaches it.\\n" + CONFIRMED=0 + fi +} ` // Removal must not take "could not look" for "not there": a sink the caller may not see would // otherwise be left routing into a topic the next step deletes. const REMOVAL_HELPERS = ` -# For an API that is off, gcloud asks whether to enable it and waits for the answer. The lookups -# below capture that question, so it would never show. Without prompts gcloud reports the API as -# disabled instead. -export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - -# exists : whether a resource is there. NOT_FOUND means it is not, and so -# does a disabled API for the host project's own resources, which cannot exist without it (a -# sink can). Any other failure (a missing permission, for example) is shown and makes this script -# end with an error, because the resource may still be there. -INCOMPLETE=0 -exists() { - local output +# remains : whether a resource is still +# there. NOT_FOUND means it is not, and so does a disabled API for the host project's own +# resources, which cannot exist without it (a sink can). Any other answer stops the script: the +# resource may still be there, and nothing that it depends on is removed before it. +remains() { + local what="$1" role="$2" output + shift 2 if output="$("$@" 2>&1)"; then return 0; fi case "$2:$output" in - *NOT_FOUND*) ;; - logging:*) echo "$output" >&2; INCOMPLETE=1 ;; - *"has not been used"* | *"is disabled"*) ;; - *) echo "$output" >&2; INCOMPLETE=1 ;; + *NOT_FOUND*) return 1 ;; + logging:*) ;; + *"has not been used"* | *"is disabled"*) return 1 ;; esac - return 1 -} - -# listed : whether the list names the -# resource. A list that fails is read like a failed describe above. -listed() { - local output - if output="$("$@" 2>/dev/null)"; then - [ -n "$output" ] - return - fi - output="$("$@" 2>&1 || true)" - case "$output" in - *"has not been used"* | *"is disabled"*) ;; - *) echo "$output" >&2; INCOMPLETE=1 ;; - esac - return 1 + stop "Couldn't check whether $what still exists." "$(fix "$output" "$role")" "$output" } # unbind : a binding that is already gone is fine. unbind() { local output if output="$("$@" 2>&1)"; then return 0; fi - case "$output" in - *"not found"*) ;; - *) echo "$output" >&2; INCOMPLETE=1 ;; - esac + case "$output" in *"not found"*) return 0 ;; esac + stop "Couldn't remove the read-only roles." "$(fix "$output" "$IAM_ROLE")" "$output" } ` -const REMOVAL_CHECK = ` -if [ "$INCOMPLETE" = 1 ]; then - echo "Not everything could be removed. See the errors above, then re-run." >&2 - exit 1 +interface AccessChecks { + /** The scope's name must be readable: true when a role the run needs includes reading it. */ + readonly scopeMustOpen: boolean + readonly billing: boolean + /** `testIamPermissions` calls: resource path, where in words, what to ask for, permissions. */ + readonly permissions: ReadonlyArray]> +} + +const openScope = (scopeType: Exclude, mustOpen: boolean): string => { + const scope = SCOPES[scopeType] + const describe = `${scope.gcloud} describe "$SCOPE_ID"` + const label = scopeType === "folder" ? "Folder" : "Organization" + return mustOpen + ? `if ! SCOPE_NAME="$(${describe} --format='value(displayName)' 2>/dev/null)"; then + stop "Can't open $SCOPE as $ACCOUNT." "Check the ID with: ${scope.list}. If it is right, this account needs ${scope.iamRole} there." "$(${describe} 2>&1 || true)" fi +ok "${label} $SCOPE_ID found ($SCOPE_NAME)" +` + : `# Not every role that can do this work may read the ${scopeType}'s name (Logs Configuration +# Writer may not), so a look that fails is not an error. +if SCOPE_NAME="$(${describe} --format='value(displayName)' 2>/dev/null)"; then + ok "${label} $SCOPE_ID found ($SCOPE_NAME)" +fi +` +} + +const PERMISSION_CHECK = ` +# allowed : asks Google which of +# the permissions this account holds, before anything is changed. No answer (no curl, no network) +# is not a refusal: the steps below then report for themselves. +CHECKED=1 +allowed() { + local resource="$1" where="$2" ask="$3" answer permission lacking="" + shift 3 + if ! answer="$(curl -fsS --max-time 15 -X POST -H 'Content-Type: application/json' \\ + -H @<(printf 'Authorization: Bearer %s\\n' "$TOKEN") \\ + --data "{\\"permissions\\":[$(printf '"%s",' "$@" | sed 's/,$//')]}" \\ + "https://cloudresourcemanager.googleapis.com/v3/$resource:testIamPermissions" 2>/dev/null)"; then + CHECKED=0 + return 0 + fi + for permission in "$@"; do + case "$answer" in *"\\"$permission\\""*) ;; *) lacking="$lacking, $permission" ;; esac + done + [ -z "$lacking" ] || stop "$ACCOUNT is missing permissions on $where: \${lacking#, }." "$ask" +} ` -const logsRemoval = (scopeType: GcpScopeType): string => ` -# ---- Logs: off. Remove what an earlier run created. ---- -# The Logs Writer grant on the host project stays: the sink wrote as the logging service agent -# of the ${scopeType}, an identity every other sink there shares. +const access = (scopeType: GcpScopeType, checks: AccessChecks): string => ` +# ---- Checking access: nothing is changed before all of it passes ---- +section "Checking access" +command -v gcloud >/dev/null 2>&1 || + stop "gcloud is not installed here." "Run this script in Cloud Shell: https://shell.cloud.google.com" +ACCOUNT="$(gcloud config get-value account 2>/dev/null || true)" +TOKEN="$(gcloud auth print-access-token 2>/dev/null || true)" +if [ -z "$ACCOUNT" ] || [ -z "$TOKEN" ]; then + stop "You are not signed in to Google Cloud." "In Cloud Shell, click Authorize when it asks. Elsewhere, run: gcloud auth login" +fi +ok "Signed in as $ACCOUNT" +${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then + stop "Can't open project $PROJECT_ID as $ACCOUNT." "Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). If it is right, this account has no access to the project." "$(gcloud projects describe "$PROJECT_ID" 2>&1 || true)" +fi +ok "${scopeType === "project" ? "Project" : "Host project"} $PROJECT_ID found ($PROJECT_NAME)" +${ + checks.billing + ? `case "$(gcloud billing projects describe "$PROJECT_ID" --format='value(billingEnabled)' 2>/dev/null || true)" in + True) ok "Billing is enabled" ;; + False) stop "Project $PROJECT_ID has no billing account." "Cloud Monitoring only answers for projects with billing. Link one at https://console.cloud.google.com/billing/linkedaccount?project=$PROJECT_ID" ;; +esac +` + : "" +}${ + checks.permissions.length === 0 + ? "" + : `${PERMISSION_CHECK}${checks.permissions + .map( + ([resource, where, ask, permissions]) => + `allowed "${resource}" "${where}" "${ask}" \\\n ${permissions.join(" ")}\n`, + ) + .join("")}[ "$CHECKED" = 0 ] || ok "$ACCOUNT has the permissions this script needs" +` +}` + +/** What the enabled sections need, asked of Google before the first change. */ +const permissionChecks = ( + scopeType: GcpScopeType, + logs: boolean, + metrics: boolean, +): AccessChecks["permissions"] => { + const scope = SCOPES[scopeType] + const administrator = "or have an administrator run this script." + const host = [ + "serviceusage.services.enable", + "resourcemanager.projects.getIamPolicy", + "resourcemanager.projects.setIamPolicy", + ...(logs + ? [ + "pubsub.topics.create", + "pubsub.topics.attachSubscription", + "pubsub.topics.setIamPolicy", + "pubsub.topics.publish", + "pubsub.subscriptions.create", + "pubsub.subscriptions.update", + ] + : []), + ...(metrics ? ["iam.serviceAccounts.create", "iam.serviceAccounts.setIamPolicy"] : []), + ] + const sink = logs ? ["logging.sinks.create", "logging.sinks.update"] : [] + const askHost = `Ask for Owner there, ${administrator}` + if (scopeType === "project") { + return [["projects/$PROJECT_ID", "project $PROJECT_ID", askHost, [...host, ...sink]]] + } + const roles = [...(logs ? ["Logs Configuration Writer"] : []), ...(metrics ? [scope.iamRole] : [])] + return [ + ["projects/$PROJECT_ID", "project $PROJECT_ID", askHost, host], + [ + `${scopeType}s/$SCOPE_ID`, + "$SCOPE", + `Ask for ${roles.join(" and ")} there, ${administrator} If the ID looks wrong, check it with: ${scope.list}`, + [...sink, ...(metrics ? [`resourcemanager.${scopeType}s.setIamPolicy`] : [])], + ], + ] +} + +const logsSetup = (scopeType: GcpScopeType): string => ` +# ---- Log forwarding: on ---- +section "Log forwarding" +DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" + +# APIs in the host project: Pub/Sub carries the log entries, Cloud Logging routes them. +run "APIs enabled (Pub/Sub, Cloud Logging)" "Couldn't enable the Pub/Sub and Cloud Logging APIs." "$OWNER" \\ + gcloud services enable pubsub.googleapis.com logging.googleapis.com --project="$PROJECT_ID" + +if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then + ok "Topic already exists" +else + run "Topic created" "Couldn't create the Pub/Sub topic." "$OWNER" \\ + gcloud pubsub topics create "$TOPIC" --project="$PROJECT_ID" +fi + +# One LogEntry per request, unwrapped. A day of retention bounds what is replayed after an +# outage, and the subscription never expires while it is idle. +SUBSCRIPTION_FLAGS=( + --project="$PROJECT_ID" + --push-endpoint="$PUSH_ENDPOINT" --push-no-wrapper + --ack-deadline=30 --message-retention-duration=1d --expiration-period=never + --min-retry-delay=10s --max-retry-delay=600s +) +if exists gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then + run "Push subscription up to date" "Couldn't update the push subscription." "$OWNER" \\ + gcloud pubsub subscriptions update "$SUBSCRIPTION" "\${SUBSCRIPTION_FLAGS[@]}" +else + run "Push subscription created" "Couldn't create the push subscription." "$OWNER" \\ + gcloud pubsub subscriptions create "$SUBSCRIPTION" --topic="$TOPIC" "\${SUBSCRIPTION_FLAGS[@]}" +fi + if exists gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then - gcloud logging sinks delete "$SINK" ${SCOPES[scopeType].sink} --quiet + if [ "$LOG_FILTER_MODE" = set ]; then + run "Log sink up to date (filter replaced)" "Couldn't update the log sink." "$SINK_ROLE" \\ + gcloud logging sinks update "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} + else + run "Log sink up to date (filter kept)" "Couldn't update the log sink." "$SINK_ROLE" \\ + gcloud logging sinks update "$SINK" "$DESTINATION" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} + fi +else + run "Log sink created" "Couldn't create the log sink." "$SINK_ROLE" \\ + gcloud logging sinks create "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} + SINK_CREATED=1 +fi + +# The sink writes as a Google-managed identity. Google requires Logs Writer for it on the project +# that holds the destination, and it needs Pub/Sub Publisher on this one topic. +if ! WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} --format='value(writerIdentity)' 2>&1)"; then + stop "Couldn't read the log sink back." "$(fix "$WRITER_IDENTITY" "$SINK_ROLE")" "$WRITER_IDENTITY" fi -# The topic stays while it is unclear whether a sink still writes to it. -if [ "$INCOMPLETE" = 0 ]; then - if exists gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then +run "Sink allowed to write to the project" "Couldn't grant the sink Logs Writer." "$OWNER" \\ + gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ + --member="$WRITER_IDENTITY" --role=roles/logging.logWriter --condition=None +run "Sink allowed to publish to the topic" "Couldn't grant the sink Pub/Sub Publisher." "$OWNER" \\ + gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ + --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher +notify "Check message sent to Maple through the topic" '"logs":true' +` + +const logsRemoval = (scopeType: GcpScopeType, notifies: boolean): string => ` +# ---- Log forwarding: off. Remove what an earlier run created. ---- +# The sink goes first, so nothing routes into a topic that is gone. The Logs Writer grant on the +# host project stays: the sink wrote as the logging service agent of the ${scopeType}, an identity +# every other sink there shares. +section "Log forwarding" +if remains "the log sink" "$SINK_ROLE" gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then + run "Log sink deleted" "Couldn't delete the log sink." "$SINK_ROLE" \\ + gcloud logging sinks delete "$SINK" ${SCOPES[scopeType].sink} --quiet + REMOVED=1 +fi +if remains "the push subscription" "$OWNER" gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then + run "Push subscription deleted" "Couldn't delete the push subscription." "$OWNER" \\ gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" --quiet - fi - if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then + REMOVED=1 +fi +if remains "the topic" "$OWNER" gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then + run "Topic deleted" "Couldn't delete the Pub/Sub topic." "$OWNER" \\ gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" --quiet - fi + REMOVED=1 fi -` +[ "$REMOVED" = 1 ] || ok "Nothing left to remove" +${notifies ? `notify "Maple notified" '"logs":false'\n` : ""}` const serviceAccountMember = '--member="serviceAccount:$SERVICE_ACCOUNT_EMAIL"' -const metricsSetup = (scopeType: GcpScopeType, mapleServiceAccountEmail: string): string => ` +const metricsSetup = (scopeType: GcpScopeType): string => ` # ---- Metrics and resources: on ---- -MAPLE_SERVICE_ACCOUNT=${sh(mapleServiceAccountEmail)} +section "Metrics and resources" # APIs in the host project, the project Maple's reads are made through: # monitoring.googleapis.com Cloud Monitoring, to read metrics # cloudasset.googleapis.com Cloud Asset Inventory, to list resources # iam.googleapis.com to create the service account below # iamcredentials.googleapis.com short-lived tokens for that account; no key is ever created -gcloud services enable monitoring.googleapis.com cloudasset.googleapis.com \\ +run "APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials)" "Couldn't enable the Cloud Monitoring, Cloud Asset and IAM APIs." "$OWNER" \\ + gcloud services enable monitoring.googleapis.com cloudasset.googleapis.com \\ iam.googleapis.com iamcredentials.googleapis.com --project="$PROJECT_ID" # A service account that an earlier run deleted (metrics switched off) fails this describe as # well and is created again. Google makes that a new account under the same name, with none of # the old one's roles, so every grant below is made for it. -if ! gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" >/dev/null 2>&1; then - gcloud iam service-accounts create "$SERVICE_ACCOUNT" --project="$PROJECT_ID" \\ +if exists gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID"; then + ok "Read-only service account already exists" +else + run "Read-only service account created" "Couldn't create the service account." "$OWNER" \\ + gcloud iam service-accounts create "$SERVICE_ACCOUNT" --project="$PROJECT_ID" \\ --display-name="Maple metrics and resource reader" + ACCOUNT_CREATED=1 fi -# A new service account can take a minute to become visible to IAM. -retry() { - for _ in 1 2 3 4 5 6; do - "$@" && return +# grant +# Google can take a minute to publish a new service account, and until then answers that it does +# not exist. Only that answer is waited out. +WAITED=0 +grant() { + local failed="$1" role="$2" output attempt + shift 2 + for attempt in 1 2 3 4 5 6 7; do + if output="$("$@" 2>&1)"; then return 0; fi + case "$output" in *"does not exist"* | *NOT_FOUND*) ;; *) break ;; esac + if [ "$attempt" = 7 ]; then break; fi + if [ "$WAITED" = 0 ]; then + printf ' \\342\\200\\246 waiting for Google to publish the new service account\\n' + WAITED=1 + fi sleep 10 done - "$@" + stop "$failed" "$(fix "$output" "$role")" "$output" } # Read-only roles on the ${scopeType}${scopeType === "project" ? "" : ", inherited by every project under it"}: @@ -219,47 +448,80 @@ retry() { # These are the narrowest predefined roles for the two calls. Both are read-only, and broader # than the calls: Cloud Asset Viewer can also read resource metadata and IAM policies. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do - retry ${SCOPES[scopeType].iam} add-iam-policy-binding "$SCOPE_ID" \\ - ${serviceAccountMember} --role="$ROLE" --condition=None >/dev/null + grant "Couldn't grant the read-only roles." "$IAM_ROLE" \\ + ${SCOPES[scopeType].gcloud} add-iam-policy-binding "$SCOPE_ID" \\ + ${serviceAccountMember} --role="$ROLE" --condition=None done - # Cloud Asset Inventory checks serviceusage.services.use on the project its API is called through. -retry gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ - ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None >/dev/null +grant "Couldn't grant the read-only roles." "$OWNER" \\ + gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ + ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None +ok "Read-only roles granted (Monitoring Viewer, Cloud Asset Viewer, Service Usage Consumer)" # Maple's own account may mint short-lived tokens for this one service account, nothing else. -# If an organization policy rejects this (domain-restricted sharing, -# constraints/iam.allowedPolicyMemberDomains), allow Maple's account there and re-run. -retry gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" \\ +grant "Couldn't let Maple use the service account." "$OWNER" \\ + gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" \\ --member="serviceAccount:$MAPLE_SERVICE_ACCOUNT" --role=roles/iam.serviceAccountTokenCreator \\ - --condition=None >/dev/null + --condition=None +ok "Maple allowed to read as that account" +notify "Maple notified" '"metrics":true' ` -const metricsRemoval = (scopeType: GcpScopeType): string => ` +const metricsRemoval = (scopeType: GcpScopeType, notifies: boolean): string => ` # ---- Metrics and resources: off. Remove what an earlier run created. ---- +section "Metrics and resources" # Looked up in the list: for 30 days after a service account is deleted, describing it answers # PERMISSION_DENIED even to an Owner, which would read as "may still be there" on every re-run. -if listed gcloud iam service-accounts list --project="$PROJECT_ID" \\ +# A list that fails is read like a failed describe. +account_remains() { + local output + if output="$("$@" 2>/dev/null)"; then + [ -n "$output" ] + return + fi + output="$("$@" 2>&1 || true)" + case "$output" in *"has not been used"* | *"is disabled"*) return 1 ;; esac + stop "Couldn't check whether the read-only service account still exists." "$(fix "$output" "$OWNER")" "$output" +} +if account_remains gcloud iam service-accounts list --project="$PROJECT_ID" \\ --filter="email=$SERVICE_ACCOUNT_EMAIL" --format='value(email)'; then - # Role bindings first: once the account is deleted they can no longer be removed by name, so - # the account stays until every one of them is gone. + # Role bindings first: once the account is deleted they can no longer be removed by name. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do - unbind ${SCOPES[scopeType].iam} remove-iam-policy-binding "$SCOPE_ID" \\ + unbind ${SCOPES[scopeType].gcloud} remove-iam-policy-binding "$SCOPE_ID" \\ ${serviceAccountMember} --role="$ROLE" --condition=None done unbind gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None - if [ "$INCOMPLETE" = 0 ]; then + ok "Read-only roles removed" + run "Read-only service account deleted" "Couldn't delete the service account." "$OWNER" \\ gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet - fi + REMOVED=1 +else + ok "Nothing left to remove" fi -` +${notifies ? `notify "Maple notified" '"metrics":false'\n` : ""}` -// Metrics are on for the connector but this deployment has lost its Google identity. Nothing is -// set up and, above all, nothing a customer set up earlier is removed. -const METRICS_UNAVAILABLE = ` -# ---- Metrics and resources: on, but not available on this Maple deployment. Left as is. ---- -` +/** + * The text that is copied: the script as a here-document for a child bash. Pasted at a prompt, + * nothing runs before the closing brace is read, a failing step ends the child and not the login + * shell, and the terminal stays the script's stdin. The first line drops the paste from the + * history of bash 5; its leading space does the same under `HISTCONTROL=ignorespace`. Nothing + * outside the here-document may be a comment: an interactive zsh runs `#` as a command. + */ +const forPaste = (marker: string, body: string) => + body.split("\n").includes(marker) + ? Effect.fail( + new IntegrationsPersistenceError({ + message: "A Google Cloud script value would end the script's here-document", + }), + ) + : Effect.succeed( + ` { [ -z "\${BASH_VERSION-}" ] || case "$(history 1)" in *${marker}*) history -d -1 ;; esac 2>/dev/null +bash /dev/fd/3 3<<'${marker}' +${body}${marker} +} +`, + ) export interface GcpSetupScriptInput extends GcpScriptTarget { /** The ingest gateway's receiver URL for this connector, including its secret. */ @@ -268,61 +530,119 @@ export interface GcpSetupScriptInput extends GcpScriptTarget { readonly mapleServiceAccountEmail: string | undefined readonly logsEnabled: boolean readonly metricsEnabled: boolean - readonly excludeGkeContainerLogs: boolean + readonly logFilter: GcpLogFilter } /** - * The script an administrator runs in Cloud Shell. Logs: a Log Router sink on the scope into a - * Pub/Sub topic whose push subscription delivers each LogEntry to Maple. Metrics and resources: a - * read-only service account Maple may impersonate. The script converges: what the connector has - * enabled is created or updated, what it has not is removed, so a re-run applies an opt-out. + * The script an administrator pastes into Cloud Shell. Logs: a Log Router sink on the scope into + * a Pub/Sub topic whose push subscription delivers each LogEntry to Maple. Metrics and resources: + * a read-only service account Maple may impersonate. The script converges: what the connector has + * enabled is created or updated, what it has not is removed, so a re-run applies an opt-out. Each + * section ends by telling Maple what it applied. */ -export const renderGcpSetupScript = (input: GcpSetupScriptInput): string => { +export const renderGcpSetupScript = (input: GcpSetupScriptInput) => { + const { scopeType, logsEnabled: logs } = input + // On for the connector while this deployment has lost its Google identity: nothing is set up + // and, above all, nothing a customer set up earlier is removed. const metrics = !input.metricsEnabled - ? { state: "off", section: metricsRemoval(input.scopeType) } + ? "off" : input.mapleServiceAccountEmail === undefined - ? { state: "on, unavailable", section: METRICS_UNAVAILABLE } - : { state: "on", section: metricsSetup(input.scopeType, input.mapleServiceAccountEmail) } - const logs = input.logsEnabled - ? { - note: "#\n# Keep this script private: PUSH_ENDPOINT contains this connector's secret.\n", - filter: ` -# ---- Edit this to change which logs are forwarded, then re-run. ---- -# Cloud Logging query language: https://cloud.google.com/logging/docs/view/logging-query-language -LOG_FILTER=${sh(gcpLogFilter(input.excludeGkeContainerLogs))} -# -------------------------------------------------------------------- -`, - section: logsSetup(input.scopeType, input.pushEndpoint), - } - : { note: "", filter: "", section: logsRemoval(input.scopeType) } - const removes = !input.logsEnabled || !input.metricsEnabled - return `#!/usr/bin/env bash -# Maple: connect a Google Cloud ${input.scopeType} to Maple. -# Logs: ${input.logsEnabled ? "on" : "off"} -# Metrics and resources: ${metrics.state} -# Run in Cloud Shell. Safe to re-run: it sets up what is on and removes what is off. -# -# You need: -${SCOPES[input.scopeType].needs} -${logs.note}set -euo pipefail -${logs.filter} -${variables(input)} -${removes ? REMOVAL_HELPERS : ""}${logs.section}${metrics.section}${removes ? REMOVAL_CHECK : ""} -echo "Maple setup complete." + ? "lost" + : "on" + const state = (on: boolean) => (on ? "on" : "off, removing") + const metricsState = + metrics === "lost" ? "not available on this Maple deployment, left as is" : state(metrics === "on") + return forPaste( + "MAPLE_SETUP_SCRIPT", + `#!/usr/bin/env bash +# Maple setup for Google Cloud ${scopeType} ${input.scopeId} +# Log forwarding: ${logs ? "on" : "off"} +# Metrics and resources: ${metrics === "lost" ? "on, unavailable" : metrics} +# Safe to run again: it sets up what is on and removes what is off. +# Keep it private: PUSH_ENDPOINT contains this connection's secret. +set -euo pipefail +export CLOUDSDK_CORE_DISABLE_PROMPTS=1 +${ + logs + ? ` +# ---- Which logs are forwarded ---- +# keep: an existing sink keeps its filter, a new sink gets LOG_FILTER. +# set: LOG_FILTER replaces the sink's filter. +# Filter syntax: https://cloud.google.com/logging/docs/view/logging-query-language +LOG_FILTER_MODE=${sh(input.logFilter === "keep" ? "keep" : "set")} +LOG_FILTER=${sh(gcpLogFilter(input.logFilter === "exclude_gke_container_logs"))} ` + : "" +}${variables(input, [ + `PUSH_ENDPOINT=${sh(input.pushEndpoint)}`, + ...(input.mapleServiceAccountEmail === undefined || metrics !== "on" + ? [] + : [`MAPLE_SERVICE_ACCOUNT=${sh(input.mapleServiceAccountEmail)}`]), + ])}SINK_CREATED=0 +ACCOUNT_CREATED=0 +REMOVED=0 +${output("Setup did not finish. Nothing needs undoing: fix this and paste the script again.")}${notify(logs)} +# exists : a look that fails for any reason falls through to the create, +# which reports for itself. +exists() { "$@" >/dev/null 2>&1; } +${logs && metrics === "on" ? "" : REMOVAL_HELPERS} +printf 'Maple setup for %s\\n' "$SCOPE" +printf ' Log forwarding %s\\n Metrics and resources %s\\n' ${sh(state(logs))} ${sh(metricsState)} +${access(scopeType, { + scopeMustOpen: metrics === "on", + billing: metrics === "on", + permissions: permissionChecks(scopeType, logs, metrics === "on"), +})}${logs ? logsSetup(scopeType) : logsRemoval(scopeType, true)}${ + metrics === "on" + ? metricsSetup(scopeType) + : metrics === "off" + ? metricsRemoval(scopeType, true) + : "" + } +if [ "$SINK_CREATED$ACCOUNT_CREATED$REMOVED" = 000 ]; then + printf '\\nDone. Everything was already in place.\\n' +elif [ "$REMOVED" = 1 ]; then + printf '\\nDone. Google Cloud matches your Maple switches.\\n' +else + printf '\\nDone. Google Cloud is set up for Maple.\\n' +fi +if [ "$CONFIRMED" = 1 ]; then printf ' Maple confirms it within a minute: %s\\n' "$MAPLE_URL"; fi +if [ "$SINK_CREATED" = 1 ]; then printf ' Logs: a new sink takes up to 10 minutes to forward its first entries.\\n'; fi +if [ "$ACCOUNT_CREATED" = 1 ]; then printf ' Metrics: the first read lands within about 10 minutes.\\n'; fi +`, + ) } -/** Removes what the setup script created. Carries no secret, so it outlives the connector. */ -export const renderGcpCleanupScript = (target: GcpScriptTarget): string => - `#!/usr/bin/env bash -# Maple: remove what the Maple setup script created for a Google Cloud ${target.scopeType}. -# Run in Cloud Shell. Safe to re-run. APIs the setup enabled stay enabled. -# -# You need: -${SCOPES[target.scopeType].needs} -set -euo pipefail +export interface GcpCleanupScriptInput extends GcpScriptTarget { + /** Set while the connector exists: the script then tells Maple that it ran. */ + readonly pushEndpoint: string | undefined +} -${variables(target)} -${REMOVAL_HELPERS}${logsRemoval(target.scopeType)}${metricsRemoval(target.scopeType)}${REMOVAL_CHECK} -echo "Maple cleanup complete." +/** Removes what the setup script created. Without a push endpoint it carries no secret. */ +export const renderGcpCleanupScript = (input: GcpCleanupScriptInput) => + forPaste( + "MAPLE_CLEANUP_SCRIPT", + `#!/usr/bin/env bash +# Maple cleanup for Google Cloud ${input.scopeType} ${input.scopeId} +# Removes what the Maple setup script created. The APIs it switched on stay on. +# Safe to run again.${input.pushEndpoint === undefined ? "" : "\n# Keep it private: PUSH_ENDPOINT contains this connection's secret."} +set -euo pipefail +export CLOUDSDK_CORE_DISABLE_PROMPTS=1 +${variables(input, input.pushEndpoint === undefined ? [] : [`PUSH_ENDPOINT=${sh(input.pushEndpoint)}`])}REMOVED=0 +${output( + "Cleanup did not finish. Fix this and paste the script again: it continues where it stopped.", +)}${REMOVAL_HELPERS} +printf 'Maple cleanup for %s\\n' "$SCOPE" +${access(input.scopeType, { scopeMustOpen: false, billing: false, permissions: [] })}${logsRemoval(input.scopeType, false)}${metricsRemoval(input.scopeType, false)}${ + input.pushEndpoint === undefined + ? "" + : ` +# Tells Maple the cleanup ran. Maple refuses this once the connection is deleted, which is fine. +${POST_REPORT} --data "$(report '"logs":false,"metrics":false')" "$PUSH_ENDPOINT" >/dev/null 2>&1 || true ` + } +printf '\\nDone. Everything the setup script created is gone.\\n' +printf ' Left in place: the APIs it switched on, and the Logs Writer role of Google'"'"'s logging\\n' +printf ' service account on %s, which other sinks share.\\n' "$PROJECT_ID" +${input.pushEndpoint === undefined ? "" : `printf ' Now disconnect in Maple: %s\\n' "$MAPLE_URL"\n`}`, + ) diff --git a/packages/db/drizzle/20261008171958_gcp_connectors/migration.sql b/packages/db/drizzle/20261008235400_gcp_connectors/migration.sql similarity index 87% rename from packages/db/drizzle/20261008171958_gcp_connectors/migration.sql rename to packages/db/drizzle/20261008235400_gcp_connectors/migration.sql index 73d9a68a2a..589967c378 100644 --- a/packages/db/drizzle/20261008171958_gcp_connectors/migration.sql +++ b/packages/db/drizzle/20261008235400_gcp_connectors/migration.sql @@ -12,6 +12,9 @@ CREATE TABLE "gcp_connectors" ( "secret_hash" text NOT NULL, "last_received_at" timestamp with time zone, "last_error" text, + "applied_logs_enabled" boolean, + "applied_metrics_enabled" boolean, + "setup_reported_at" timestamp with time zone, "created_by" text NOT NULL, "created_at" timestamp with time zone NOT NULL, "updated_at" timestamp with time zone NOT NULL diff --git a/packages/db/drizzle/20261008171958_gcp_connectors/snapshot.json b/packages/db/drizzle/20261008235400_gcp_connectors/snapshot.json similarity index 99% rename from packages/db/drizzle/20261008171958_gcp_connectors/snapshot.json rename to packages/db/drizzle/20261008235400_gcp_connectors/snapshot.json index e1e30a3018..857ceb390f 100644 --- a/packages/db/drizzle/20261008171958_gcp_connectors/snapshot.json +++ b/packages/db/drizzle/20261008235400_gcp_connectors/snapshot.json @@ -1,7 +1,7 @@ { "version": "8", "dialect": "postgres", - "id": "c2de92d1-c56e-4b42-b7af-16fa811a694e", + "id": "7b6e40a7-3210-4afa-9815-9f703b97f891", "prevIds": [ "a9ce02f8-139d-4eb9-bf09-315c5936b0fa" ], @@ -4260,6 +4260,45 @@ "schema": "public", "table": "gcp_connectors" }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_logs_enabled", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_metrics_enabled", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "setup_reported_at", + "entityType": "columns", + "schema": "public", + "table": "gcp_connectors" + }, { "type": "text", "typeSchema": null, diff --git a/packages/db/src/schema/gcp-connectors.ts b/packages/db/src/schema/gcp-connectors.ts index 5e9d264755..74eb1ead13 100644 --- a/packages/db/src/schema/gcp-connectors.ts +++ b/packages/db/src/schema/gcp-connectors.ts @@ -5,7 +5,8 @@ import { boolean, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-c // organization. `project_id` is the host project, where Maple's own resources live (Pub/Sub topic, // subscription, service account); for a project scope it equals `scope_id`. The ingest gateway's // `/v1/logpush/gcp/...` receiver authenticates a push by `id` + `secret_hash` and writes -// `last_received_at` / `last_error`. `metrics_enabled` covers metrics and resource collection. +// `last_received_at` / `last_error` and the setup report. `metrics_enabled` covers metrics and +// resource collection. export const gcpConnectors = pgTable( "gcp_connectors", { @@ -22,6 +23,11 @@ export const gcpConnectors = pgTable( secretHash: text("secret_hash").notNull(), lastReceivedAt: timestamp("last_received_at", { withTimezone: true, mode: "date" }), lastError: text("last_error"), + // What the latest setup script run reported it set up in Google Cloud, written by the + // receiver. Null until a run reports: the switches above say what should be there. + appliedLogsEnabled: boolean("applied_logs_enabled"), + appliedMetricsEnabled: boolean("applied_metrics_enabled"), + setupReportedAt: timestamp("setup_reported_at", { withTimezone: true, mode: "date" }), createdBy: text("created_by").notNull(), createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).notNull(), updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }).notNull(), diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts index ee6f1bc98a..9d34bf0b6a 100644 --- a/packages/domain/src/http/v2/integrations-gcp.ts +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -1,6 +1,6 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api" import { Schema } from "effect" -import { GcpConnectorId, GcpProjectId, GcpResourceNumber, GcpScopeType } from "../../primitives" +import { GcpConnectorId, GcpLogFilter, GcpProjectId, GcpResourceNumber, GcpScopeType } from "../../primitives" import { GcpMetricsUnavailableError, GcpScopeAlreadyConnectedError, @@ -34,6 +34,9 @@ const connectorExample = { created_at: "2026-10-01T12:00:00.000Z", last_log_received_at: "2026-10-08T09:12:00.000Z", last_log_error: null, + applied_logs_enabled: true, + applied_metrics_enabled: true, + setup_reported_at: "2026-10-01T12:05:00.000Z", } as const export const V2GcpConnector = Schema.Struct({ @@ -73,6 +76,20 @@ export const V2GcpConnector = Schema.Struct({ description: "Why the most recent log push was rejected, or `null`.", examples: [null], }), + applied_logs_enabled: Schema.NullOr(Schema.Boolean).annotate({ + description: + "Whether log forwarding is set up in Google Cloud, as the latest setup script run reported it. `null` until a run reports. While it differs from `logs_enabled`, the setup script has to run again.", + examples: [true], + }), + applied_metrics_enabled: Schema.NullOr(Schema.Boolean).annotate({ + description: + "Whether metrics and resource access is set up in Google Cloud, as the latest setup script run reported it. `null` until a run reports. While it differs from `metrics_enabled`, the setup script has to run again.", + examples: [true], + }), + setup_reported_at: Schema.NullOr(Timestamp).annotate({ + description: + "When a setup or cleanup script run last reported to Maple, or `null`. A run reports within seconds of finishing, before its first log or metric arrives.", + }), }).annotate({ identifier: "GcpConnector", title: "Google Cloud connector", @@ -174,17 +191,21 @@ export const V2GcpUpdateConnectorRequest = Schema.Struct({ export type V2GcpUpdateConnectorRequest = Schema.Schema.Type export const V2GcpSetupScriptsRequest = Schema.Struct({ - exclude_gke_container_logs: Schema.optionalKey(Schema.Boolean).annotate({ + log_filter: Schema.optionalKey(GcpLogFilter).annotate({ description: - "Leave GKE container logs out of the log filter. Set it when pods already ship their logs to Maple through an OpenTelemetry collector, so they are not ingested twice. Defaults to `false`.", - examples: [false], + "The log filter the setup script carries. `keep` (the default): a sink that already exists keeps its filter, and a new one gets Maple's default. `default`: the script sets Maple's default, which leaves out Data Access audit logs and load balancer health checks. `exclude_gke_container_logs`: the script sets the default without GKE container logs, for pods that already ship their logs to Maple through an OpenTelemetry collector.", + examples: ["keep"], }), }).annotate({ identifier: "GcpSetupScriptsRequest", title: "Google Cloud setup scripts request", description: "Options for rendering a connector's scripts.", - examples: [wireExample({ exclude_gke_container_logs: true })], + examples: [wireExample({ log_filter: "exclude_gke_container_logs" })], }) + +// What a script field holds: the script as a here-document for a bash process of its own. +const SCRIPT_EXAMPLE = + " { …\nbash /dev/fd/3 3<<'MAPLE_SETUP_SCRIPT'\n#!/usr/bin/env bash\n…\nMAPLE_SETUP_SCRIPT\n}\n" export type V2GcpSetupScriptsRequest = Schema.Schema.Type export const V2GcpSetupScripts = Schema.Struct({ @@ -193,12 +214,13 @@ export const V2GcpSetupScripts = Schema.Struct({ }), setup_script: Schema.String.annotate({ description: - "A bash script for Cloud Shell. It sets up what the connector has enabled and removes what it has not, so re-run it after changing `logs_enabled` or `metrics_enabled`. With logs enabled it embeds the connector's push secret: treat it as a credential.", - examples: ["#!/usr/bin/env bash\n…"], + "Text to paste into Cloud Shell: a bash script, wrapped so that it runs in a bash process of its own. It sets up what the connector has enabled, removes what it has not, and tells Maple what it applied, so run it again after changing `logs_enabled` or `metrics_enabled`. It embeds the connector's push secret: treat it as a credential.", + examples: [SCRIPT_EXAMPLE], }), cleanup_script: Schema.String.annotate({ - description: "A bash script that removes everything the setup script created. Carries no secret.", - examples: ["#!/usr/bin/env bash\n…"], + description: + "Text to paste into Cloud Shell that removes everything the setup script created and tells Maple that it ran. It embeds the connector's push secret: treat it as a credential.", + examples: [SCRIPT_EXAMPLE], }), }).annotate({ identifier: "GcpSetupScripts", @@ -207,8 +229,8 @@ export const V2GcpSetupScripts = Schema.Struct({ examples: [ wireExample({ object: "gcp_connector.setup_scripts", - setup_script: "#!/usr/bin/env bash\n…", - cleanup_script: "#!/usr/bin/env bash\n…", + setup_script: SCRIPT_EXAMPLE, + cleanup_script: SCRIPT_EXAMPLE, }), ], }) @@ -224,8 +246,8 @@ export const V2GcpConnectorDeleteResponse = Schema.Struct({ }), cleanup_script: Schema.String.annotate({ description: - "A bash script that removes the resources the setup script created. Maple has no write access to Google Cloud, so an administrator runs it.", - examples: ["#!/usr/bin/env bash\n…"], + "Text to paste into Cloud Shell that removes the resources the setup script created. Maple has no write access to Google Cloud, so an administrator runs it. The connector is gone, so this copy carries no secret.", + examples: [SCRIPT_EXAMPLE], }), }).annotate({ identifier: "GcpConnectorDeleteResponse", @@ -236,7 +258,7 @@ export const V2GcpConnectorDeleteResponse = Schema.Struct({ id: connectorExample.id, object: "gcp_connector", deleted: true, - cleanup_script: "#!/usr/bin/env bash\n…", + cleanup_script: SCRIPT_EXAMPLE, }), ], }) @@ -261,7 +283,7 @@ export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration identifier: "getGcpIntegration", summary: "Retrieve Google Cloud integration status", description: - "Returns the organization's Google Cloud connectors with the time and outcome of their latest log push, and whether metrics collection is available. Requires the `integrations:read` scope.", + "Returns the organization's Google Cloud connectors with what their setup script last reported and the time and outcome of their latest log push, and whether metrics collection is available. Requires the `integrations:read` scope.", }), ), ) @@ -317,7 +339,7 @@ export class V2GcpIntegrationsApiGroup extends HttpApiGroup.make("gcpIntegration identifier: "renderGcpConnectorSetupScripts", summary: "Render a connector's setup and cleanup scripts", description: - "Returns the `gcloud` scripts for one connector. Rendering changes nothing, but the setup script carries the connector's push secret, so this requires an org-admin role and the `integrations:write` scope.", + "Returns the `gcloud` scripts for one connector. Rendering changes nothing, but both scripts carry the connector's push secret, so this requires an org-admin role and the `integrations:write` scope.", }), ), ) diff --git a/packages/primitives/src/index.ts b/packages/primitives/src/index.ts index c2a3dfd0b7..351f443f0c 100644 --- a/packages/primitives/src/index.ts +++ b/packages/primitives/src/index.ts @@ -122,6 +122,13 @@ export type GcpResourceNumber = Schema.Schema.Type export const GcpScopeType = Schema.Literals(["project", "folder", "organization"]) export type GcpScopeType = Schema.Schema.Type +/** + * The log filter a rendered setup script carries. `keep` leaves an existing sink's filter alone, + * the others replace it. + */ +export const GcpLogFilter = Schema.Literals(["keep", "default", "exclude_gke_container_logs"]) +export type GcpLogFilter = Schema.Schema.Type + export const AlertDestinationId = MapleUuidId("@maple/AlertDestinationId", "Alert Destination ID") export type AlertDestinationId = Schema.Schema.Type From 851d715beaaee1b7ad2bb33f012d1a97ec83c68a Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 02:35:43 +0200 Subject: [PATCH 12/21] fix(integrations): Google Cloud script remedies for a disabled API, a slow new account and a host-project binding A sink's writer identity is read without gcloud's stderr, Google's answer is printed without the push secret, only gcloud's own wording counts as a binding that is already gone, and a refused removal names the role of the resource it was refused on. --- .../integrations/gcp/setup-scripts.test.ts | 52 ++++++++++++++++--- .../integrations/gcp/setup-scripts.ts | 47 +++++++++++------ .../domain/src/http/v2/integrations-gcp.ts | 6 +-- 3 files changed, 80 insertions(+), 25 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index 67d9a41633..ff7858a5e0 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -85,12 +85,13 @@ const cleanup = (scope: Scope, pushEndpoint?: string) => // A `gcloud` that records its commands, and each argument on its own line so a value split by // bad quoting shows. It talks on stderr like the real one, which a script must not pass on. // `describe` answers as the test says: found, missing, or denied, and so does the service account -// list (a missing account is an empty list). `GCLOUD_FAIL` refuses the commands that contain it. +// list (a missing account is an empty list). `GCLOUD_FAIL` refuses the commands that contain it, +// in the words of `GCLOUD_ANSWER` when the test gives some. const FAKE_GCLOUD = `#!/usr/bin/env bash echo "$*" >> "$GCLOUD_LOG" printf "%s\\n" "$@" >> "$GCLOUD_LOG.args" denied() { - echo "ERROR: (gcloud) PERMISSION_DENIED: Permission denied on resource (or it may not exist)." >&2 + echo "\${GCLOUD_ANSWER:-ERROR: (gcloud) PERMISSION_DENIED: Permission denied on resource (or it may not exist).}" >&2 exit 1 } # For a disabled API gcloud asks whether to enable it, unless prompts are off. @@ -109,7 +110,7 @@ case "$*" in "projects describe"*) echo "Acme Production" ;; "organizations describe"* | "resource-manager folders describe"*) echo "acme.com" ;; "billing projects describe"*) echo "$GCLOUD_BILLING" ;; - *"sinks describe"*"--format"*) echo "${WRITER}" ;; + *"sinks describe"*"--format"*) echo "WARNING: gcloud says something on stderr." >&2; echo "${WRITER}" ;; *"service-accounts list"*) case "$GCLOUD_DESCRIBE" in found) echo "${ACCOUNT}" ;; @@ -182,8 +183,9 @@ interface RunOptions { describe?: "found" | "missing" | "denied" | "api_off" unbind?: "ok" | "absent" | "denied" gcloud?: string - /** A substring of the gcloud commands to refuse. */ + /** A substring of the gcloud commands to refuse, and Google's words for it. */ fail?: string + answer?: string account?: string billing?: "True" | "False" /** A permission the account does not hold, or `unanswered` when Google cannot be asked. */ @@ -216,6 +218,7 @@ const run = (script: string, options: RunOptions = {}) => { GCLOUD_DESCRIBE: options.describe ?? "missing", GCLOUD_UNBIND: options.unbind ?? "ok", GCLOUD_FAIL: options.fail ?? "", + GCLOUD_ANSWER: options.answer ?? "", GCLOUD_ACCOUNT: options.account ?? "jane@acme.com", GCLOUD_BILLING: options.billing ?? "True", CURL_LOG: join(dir, "curl.log"), @@ -580,6 +583,27 @@ ${UNFINISHED}`), ).toBe(true) }) + it("reads Google's other refusals for what they are, and never prints the secret back", () => { + const script = setup(setupInput("project", { logs: true, metrics: true })) + const propagating = run(script, { + fail: "topics create", + answer: "ERROR: (gcloud) PERMISSION_DENIED: Cloud Pub/Sub API has not been used in project 1 before or it is disabled.", + }) + expect(propagating.stdout).toContain( + "What to do: Google is still switching an API on, or the API is off. Wait a minute and paste the script again.", + ) + + const quoted = run(script, { + fail: "subscriptions create", + answer: `ERROR: (gcloud) INVALID_ARGUMENT: Invalid push endpoint given (endpoint=${PUSH_ENDPOINT}).`, + }) + expect(quoted.stdout).toContain("?secret=HIDDEN).") + expect(quoted.stdout).not.toContain(SECRET) + expect(quoted.stdout).toContain( + "What to do: Read Google's answer above. If it is unclear, send this output to support@maple.dev.", + ) + }) + it("names the role of the scope for a refused grant there", () => { const { stdout } = run(setup(setupInput("organization", { logs: false, metrics: true })), { fail: "organizations add-iam-policy-binding", @@ -733,7 +757,9 @@ esac ...setupInput("organization", { logs: true, metrics: true }), mapleServiceAccountEmail: undefined, }) - expect(script).toContain("Metrics and resources: on, unavailable") + expect(script).toContain( + "# Metrics and resources: not available on this Maple deployment, left as is\n", + ) const { status, stdout, commands } = run(script, { describe: "found" }) expect(status).toBe(0) expect(stdout).toContain( @@ -763,6 +789,18 @@ esac expect(denied.stdout).toContain( "\nCleanup did not finish. Fix this and paste the script again: it continues where it stopped.\n", ) + + // The host project's binding is the Owner's to remove, and only gcloud's own "not found" + // means a binding is gone. + const hostDenied = run(script, { describe: "found", fail: "projects remove-iam-policy-binding" }) + expect(hostDenied.stdout).toContain("What to do: jane@acme.com needs Owner on project acme-host.") + const unknown = run(script, { + describe: "found", + fail: "remove-iam-policy-binding", + answer: "ERROR: (gcloud) NOT_FOUND: Requested entity was not found.", + }) + expect(unknown.commands).not.toContainEqual(expect.stringContaining("service-accounts delete")) + expect(unknown.status).toBe(1) }) it("keeps the topic when it cannot tell whether the sink is gone", () => { @@ -839,8 +877,8 @@ esac it("carries no secret and reports nothing once the connector is gone", () => { const script = cleanup(SCOPES.project) - expect(script).not.toContain("secret") - expect(script).not.toContain("PUSH_ENDPOINT") + expect(script).not.toContain("maple_gcp_") + expect(script).not.toContain("PUSH_ENDPOINT=") const { status, stdout, curl } = run(script, { describe: "found" }) expect(status).toBe(0) expect(curl).toEqual([]) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 7fa6e7cb86..02a75bb433 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -9,7 +9,10 @@ import type { } from "@maple/domain/primitives" import { Effect } from "effect" -/** Single-quote a value for bash. Every value that reaches a script goes through here. */ +/** + * Single-quote a value for bash. Every value a script uses goes through here; the scope named in + * its header comment is an id the API has pattern-checked. + */ const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` const DOCS_URL = "https://maple.dev/docs/integrations/gcp" @@ -96,7 +99,11 @@ ok() { printf ' \\342\\234\\223 %s\\n' "$1"; } # stop [Google's answer] stop() { printf ' \\342\\234\\227 %s\\n' "$1" - if [ -n "\${3-}" ]; then printf '\\n'; printf '%s\\n' "$3" | sed 's/^/ /'; fi + # Google can quote the push endpoint back, and this output may be sent to support. + if [ -n "\${3-}" ]; then + printf '\\n' + printf '%s\\n' "$3" | sed -e 's/secret=[^ )&"]*/secret=HIDDEN/g' -e 's/^/ /' + fi printf '\\n What to do: %s\\n' "$2" printf '\\n%s\\n' ${sh(unfinished)} exit 1 @@ -105,10 +112,15 @@ stop() { # fix : what to do about a refused step. fix() { case "$1" in - *"requires billing"* | *BILLING_DISABLED*) + *"requires billing"* | *BILLING_DISABLED* | *"Billing must be enabled"* | *BILLING_NOT_FOUND*) printf '%s' "Project $PROJECT_ID has no active billing account. Link one at https://console.cloud.google.com/billing/linkedaccount?project=$PROJECT_ID" ;; *allowedPolicyMember* | *"permitted customer"*) printf '%s' "An organization policy (domain restricted sharing) blocks this grant. Ask an Organization Policy Administrator to lift it for project $PROJECT_ID while this script runs. See ${DOCS_URL}#domain-restricted-sharing" ;; + # Google words these two as a denial too, so they are read before the missing role. + *"has not been used"* | *"is disabled"* | *SERVICE_DISABLED*) + printf '%s' "Google is still switching an API on, or the API is off. Wait a minute and paste the script again." ;; + *"ervice account"*"does not exist"*) + printf '%s' "Google has not published the new service account yet. Wait a minute and paste the script again." ;; *PERMISSION_DENIED* | *"does not have permission"* | *"denied on resource"*) printf '%s' "$ACCOUNT needs $2." ;; *) printf '%s' "Read Google's answer above. If it is unclear, send this output to support@maple.dev." ;; @@ -152,7 +164,7 @@ notify() { : `${POST_REPORT} --data "$(report "$2")" "$PUSH_ENDPOINT"` } >/dev/null 2>&1; then ok "$1" - else + elif [ "$CONFIRMED" = 1 ]; then printf " \\342\\200\\242 Couldn't reach Maple to confirm. Maple shows the connection as pending until a later run reaches it.\\n" CONFIRMED=0 fi @@ -164,8 +176,9 @@ notify() { const REMOVAL_HELPERS = ` # remains : whether a resource is still # there. NOT_FOUND means it is not, and so does a disabled API for the host project's own -# resources, which cannot exist without it (a sink can). Any other answer stops the script: the -# resource may still be there, and nothing that it depends on is removed before it. +# resources: setup switches their APIs on before it creates them (a sink can exist without). +# Any other answer stops the script: the resource may still be there, and nothing that it +# depends on is removed before it. remains() { local what="$1" role="$2" output shift 2 @@ -178,12 +191,14 @@ remains() { stop "Couldn't check whether $what still exists." "$(fix "$output" "$role")" "$output" } -# unbind : a binding that is already gone is fine. +# unbind : a binding that is +# already gone is fine. unbind() { - local output + local role="$1" output + shift if output="$("$@" 2>&1)"; then return 0; fi - case "$output" in *"not found"*) return 0 ;; esac - stop "Couldn't remove the read-only roles." "$(fix "$output" "$IAM_ROLE")" "$output" + case "$output" in *"Policy binding"*"not found"*) return 0 ;; esac + stop "Couldn't remove the read-only roles." "$(fix "$output" "$role")" "$output" } ` @@ -276,6 +291,7 @@ const permissionChecks = ( logs: boolean, metrics: boolean, ): AccessChecks["permissions"] => { + if (!logs && !metrics) return [] const scope = SCOPES[scopeType] const administrator = "or have an administrator run this script." const host = [ @@ -359,7 +375,8 @@ fi # The sink writes as a Google-managed identity. Google requires Logs Writer for it on the project # that holds the destination, and it needs Pub/Sub Publisher on this one topic. -if ! WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} --format='value(writerIdentity)' 2>&1)"; then +if ! WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} --format='value(writerIdentity)' 2>/dev/null)"; then + WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} 2>&1 || true)" stop "Couldn't read the log sink back." "$(fix "$WRITER_IDENTITY" "$SINK_ROLE")" "$WRITER_IDENTITY" fi run "Sink allowed to write to the project" "Couldn't grant the sink Logs Writer." "$OWNER" \\ @@ -487,10 +504,10 @@ if account_remains gcloud iam service-accounts list --project="$PROJECT_ID" \\ --filter="email=$SERVICE_ACCOUNT_EMAIL" --format='value(email)'; then # Role bindings first: once the account is deleted they can no longer be removed by name. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do - unbind ${SCOPES[scopeType].gcloud} remove-iam-policy-binding "$SCOPE_ID" \\ + unbind "$IAM_ROLE" ${SCOPES[scopeType].gcloud} remove-iam-policy-binding "$SCOPE_ID" \\ ${serviceAccountMember} --role="$ROLE" --condition=None done - unbind gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ + unbind "$OWNER" gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None ok "Read-only roles removed" run "Read-only service account deleted" "Couldn't delete the service account." "$OWNER" \\ @@ -508,7 +525,7 @@ ${notifies ? `notify "Maple notified" '"metrics":false'\n` : ""}` * history of bash 5; its leading space does the same under `HISTCONTROL=ignorespace`. Nothing * outside the here-document may be a comment: an interactive zsh runs `#` as a command. */ -const forPaste = (marker: string, body: string) => +const forPaste = (marker: string, body: string): Effect.Effect => body.split("\n").includes(marker) ? Effect.fail( new IntegrationsPersistenceError({ @@ -557,7 +574,7 @@ export const renderGcpSetupScript = (input: GcpSetupScriptInput) => { `#!/usr/bin/env bash # Maple setup for Google Cloud ${scopeType} ${input.scopeId} # Log forwarding: ${logs ? "on" : "off"} -# Metrics and resources: ${metrics === "lost" ? "on, unavailable" : metrics} +# Metrics and resources: ${metrics === "lost" ? metricsState : metrics} # Safe to run again: it sets up what is on and removes what is off. # Keep it private: PUSH_ENDPOINT contains this connection's secret. set -euo pipefail diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts index 9d34bf0b6a..6acab4a9f1 100644 --- a/packages/domain/src/http/v2/integrations-gcp.ts +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -78,12 +78,12 @@ export const V2GcpConnector = Schema.Struct({ }), applied_logs_enabled: Schema.NullOr(Schema.Boolean).annotate({ description: - "Whether log forwarding is set up in Google Cloud, as the latest setup script run reported it. `null` until a run reports. While it differs from `logs_enabled`, the setup script has to run again.", + "Whether log forwarding is set up in Google Cloud, as the latest setup script run reported it. `null` until a run reports. The setup script has to run while `logs_enabled` is `true` and this is not, or `logs_enabled` is `false` and this is `true`.", examples: [true], }), applied_metrics_enabled: Schema.NullOr(Schema.Boolean).annotate({ description: - "Whether metrics and resource access is set up in Google Cloud, as the latest setup script run reported it. `null` until a run reports. While it differs from `metrics_enabled`, the setup script has to run again.", + "Whether metrics and resource access is set up in Google Cloud, as the latest setup script run reported it. `null` until a run reports. The setup script has to run while `metrics_enabled` is `true` and this is not, or `metrics_enabled` is `false` and this is `true`.", examples: [true], }), setup_reported_at: Schema.NullOr(Timestamp).annotate({ @@ -202,11 +202,11 @@ export const V2GcpSetupScriptsRequest = Schema.Struct({ description: "Options for rendering a connector's scripts.", examples: [wireExample({ log_filter: "exclude_gke_container_logs" })], }) +export type V2GcpSetupScriptsRequest = Schema.Schema.Type // What a script field holds: the script as a here-document for a bash process of its own. const SCRIPT_EXAMPLE = " { …\nbash /dev/fd/3 3<<'MAPLE_SETUP_SCRIPT'\n#!/usr/bin/env bash\n…\nMAPLE_SETUP_SCRIPT\n}\n" -export type V2GcpSetupScriptsRequest = Schema.Schema.Type export const V2GcpSetupScripts = Schema.Struct({ object: Schema.Literal("gcp_connector.setup_scripts").annotate({ From eece8d71f5a95bb2106861157fd8428501ec2bab Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 02:45:03 +0200 Subject: [PATCH 13/21] fix(integrations): Google Cloud script for callers with fewer rights, and a quieter default log filter The script switches on only the APIs that are off, so someone who may not switch APIs on can run it once they are, and it makes sure the Cloud Resource Manager API is on for a service-account caller. The default filter also leaves out Kubernetes lease renewals and VM serial console output. The summary says that entries logged before a new sink is live are not forwarded later. --- .../routes/v2/integrations-gcp.http.test.ts | 2 +- .../integrations/gcp/setup-scripts.test.ts | 74 +++++++++++++++--- .../integrations/gcp/setup-scripts.ts | 77 +++++++++++++++---- .../domain/src/http/v2/integrations-gcp.ts | 2 +- 4 files changed, 128 insertions(+), 27 deletions(-) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index d337006366..42228a0957 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -221,7 +221,7 @@ describe("v2 gcp integration over HTTP", () => { expect(both.body.setup_script).toContain("SCOPE_ID='123456789012'") expect(both.body.setup_script).toContain('--folder="$SCOPE_ID" --include-children') expect(both.body.setup_script).toContain( - `LOG_FILTER_MODE='set'\nLOG_FILTER='NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT resource.type="k8s_container"'`, + `LOG_FILTER_MODE='set'\nLOG_FILTER='NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update" AND NOT logName:"serialconsole.googleapis.com" AND NOT resource.type="k8s_container"'`, ) expect(both.body.setup_script).toContain( 'gcloud resource-manager folders add-iam-policy-binding "$SCOPE_ID"', diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index ff7858a5e0..3103b9ff19 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -110,6 +110,7 @@ case "$*" in "projects describe"*) echo "Acme Production" ;; "organizations describe"* | "resource-manager folders describe"*) echo "acme.com" ;; "billing projects describe"*) echo "$GCLOUD_BILLING" ;; + "services list"*) echo "$GCLOUD_APIS" ;; *"sinks describe"*"--format"*) echo "WARNING: gcloud says something on stderr." >&2; echo "${WRITER}" ;; *"service-accounts list"*) case "$GCLOUD_DESCRIBE" in @@ -188,6 +189,8 @@ interface RunOptions { answer?: string account?: string billing?: "True" | "False" + /** The APIs that are on in the host project. None unless the test says so. */ + apis?: ReadonlyArray /** A permission the account does not hold, or `unanswered` when Google cannot be asked. */ lacking?: string report?: "ok" | "unreachable" @@ -221,6 +224,7 @@ const run = (script: string, options: RunOptions = {}) => { GCLOUD_ANSWER: options.answer ?? "", GCLOUD_ACCOUNT: options.account ?? "jane@acme.com", GCLOUD_BILLING: options.billing ?? "True", + GCLOUD_APIS: (options.apis ?? []).join("\n"), CURL_LOG: join(dir, "curl.log"), CURL_IAM: options.lacking === "unanswered" ? "unanswered" : "ok", CURL_LACKING: options.lacking ?? "", @@ -240,7 +244,13 @@ const run = (script: string, options: RunOptions = {}) => { } } +const API_LOOKUP = "services list --enabled --project=acme-host --format=value(config.name)" +// Tried before anything is read, for a caller whose gcloud calls are counted against this project. +const RESOURCE_MANAGER_ON = "services enable cloudresourcemanager.googleapis.com --project=acme-host" + const isAccessCheck = (command: string) => + command === API_LOOKUP || + command === RESOURCE_MANAGER_ON || /^(config get-value|auth print-access-token|projects describe|organizations describe|resource-manager folders describe|billing projects describe)/.test( command, ) @@ -258,7 +268,7 @@ const publish = (members: string) => ) const logsSetupCommands = (scope: Scope) => [ - "services enable pubsub.googleapis.com logging.googleapis.com --project=acme-host", + "services enable pubsub.googleapis.com logging.googleapis.com cloudresourcemanager.googleapis.com --project=acme-host", `pubsub topics describe ${NAME} --project=acme-host`, `pubsub topics create ${NAME} --project=acme-host`, `pubsub subscriptions describe ${NAME} --project=acme-host`, @@ -272,7 +282,7 @@ const logsSetupCommands = (scope: Scope) => [ ] const metricsSetupCommands = (scope: Scope) => [ - "services enable monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com iamcredentials.googleapis.com --project=acme-host", + "services enable monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com iamcredentials.googleapis.com cloudresourcemanager.googleapis.com --project=acme-host", `iam service-accounts describe ${ACCOUNT} --project=acme-host`, `iam service-accounts create ${NAME} --project=acme-host --display-name=Maple metrics and resource reader`, `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, @@ -323,9 +333,14 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { expect(access).toEqual([ "config get-value account", "auth print-access-token", + API_LOOKUP, + RESOURCE_MANAGER_ON, ...scope.open, "projects describe acme-host --format=value(name)", "billing projects describe acme-host --format=value(billingEnabled)", + // Each section asks which of its APIs are off. + API_LOOKUP, + API_LOOKUP, ]) expect(stdout).toBe(`Maple setup for ${scopeType} ${scope.id} Log forwarding on @@ -338,7 +353,7 @@ ${[...scope.found, ` ✓ ${host} acme-host found (Acme Production)`].join("\n") ✓ jane@acme.com has the permissions this script needs Log forwarding - ✓ APIs enabled (Pub/Sub, Cloud Logging) + ✓ APIs enabled (Pub/Sub, Cloud Logging, Cloud Resource Manager) ✓ Topic created ✓ Push subscription created ✓ Log sink created @@ -347,7 +362,7 @@ Log forwarding ✓ Check message sent to Maple through the topic Metrics and resources - ✓ APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials) + ✓ APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials, Cloud Resource Manager) ✓ Read-only service account created ✓ Read-only roles granted (Monitoring Viewer, Cloud Asset Viewer, Service Usage Consumer) ✓ Maple allowed to read as that account @@ -355,7 +370,8 @@ Metrics and resources Done. Google Cloud is set up for Maple. Maple confirms it within a minute: ${MAPLE_URL} - Logs: a new sink takes up to 10 minutes to forward its first entries. + Logs: a new sink can take about 10 minutes to start forwarding. What is logged before + it does is not forwarded later. Metrics: the first read lands within about 10 minutes. `) }) @@ -483,10 +499,11 @@ describe("the text that is copied", () => { const { stdout } = run(script, { fail: "sinks create", shell: ["--norc", "--noprofile", "-i"], - then: "echo STATUS=$? ALIVE; set -o | grep -E 'errexit|nounset'; history | grep -c 'MAPLE_SETUP_SCRIP[T]'\n", + // The next command to fail must not close the shell either. + then: "echo STATUS=$? ALIVE; false; echo AFTER=$?; set -o | grep -E 'errexit|nounset'; history | grep -c 'MAPLE_SETUP_SCRIP[T]'\n", }) expect(stdout).toContain(UNFINISHED) - expect(stdout).toContain("STATUS=1 ALIVE") + expect(stdout).toContain("STATUS=1 ALIVE\nAFTER=1\n") expect(stdout).toMatch(/errexit\s+off\nnounset\s+off/) // bash 5 can name the running command's history entry; older ones keep the paste. const bashMajor = Number( @@ -522,7 +539,7 @@ describe("renderGcpSetupScript", () => { expect(commands).not.toContainEqual(expect.stringContaining("--log-filter")) expect(stdout).toContain(` Log forwarding - ✓ APIs enabled (Pub/Sub, Cloud Logging) + ✓ APIs enabled (Pub/Sub, Cloud Logging, Cloud Resource Manager) ✓ Topic already exists ✓ Push subscription up to date ✓ Log sink up to date (filter kept) @@ -531,7 +548,7 @@ Log forwarding ✓ Check message sent to Maple through the topic Metrics and resources - ✓ APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials) + ✓ APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials, Cloud Resource Manager) ✓ Read-only service account already exists ✓ Read-only roles granted (Monitoring Viewer, Cloud Asset Viewer, Service Usage Consumer) ✓ Maple allowed to read as that account @@ -571,7 +588,7 @@ Done. Everything was already in place. expect( stdout.endsWith(` Log forwarding - ✓ APIs enabled (Pub/Sub, Cloud Logging) + ✓ APIs enabled (Pub/Sub, Cloud Logging, Cloud Resource Manager) ✓ Topic created ✓ Push subscription created ✗ Couldn't create the log sink. @@ -650,6 +667,39 @@ ${UNFINISHED}`) expect(logsOnly.stdout).not.toContain("Organization 123456789012 found") }) + it("switches on only the APIs that are off, so a caller who may not switch any on can still run it", () => { + const script = setup(setupInput("project", { logs: true, metrics: true })) + const all = [ + "pubsub.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "cloudasset.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "cloudresourcemanager.googleapis.com", + ] + const allOn = run(script, { apis: all, fail: "services enable" }) + expect(allOn.status).toBe(0) + expect(allOn.commands).not.toContainEqual(expect.stringContaining("services enable")) + expect(allOn.access).not.toContain(RESOURCE_MANAGER_ON) + expect(allOn.stdout).toContain(" ✓ APIs enabled (Pub/Sub, Cloud Logging, Cloud Resource Manager)\n") + + // One is off: only that one is switched on, and a refusal names it and the role. + const oneOff = run(script, { apis: all.filter((api) => api !== "iam.googleapis.com") }) + expect(oneOff.commands.filter((command) => command.startsWith("services enable"))).toEqual([ + "services enable iam.googleapis.com --project=acme-host", + ]) + const refused = run(script, { + apis: all.filter((api) => api !== "pubsub.googleapis.com"), + fail: "services enable", + }) + expect(refused.status).toBe(1) + expect(refused.stdout).toContain(` ✗ Couldn't switch on: pubsub.googleapis.com.`) + expect(refused.stdout).toContain( + "What to do: jane@acme.com needs Service Usage Admin on project acme-host.", + ) + }) + it("stops before the first change without billing, a sign-in, or gcloud", () => { const script = setup(setupInput("project", { logs: true, metrics: true })) const unbilled = run(script, { billing: "False" }) @@ -899,9 +949,9 @@ esac ) }) - it("excludes data-access audit logs and health-check probes by default, GKE containers on request", () => { + it("leaves out high-volume noise by default, and GKE containers on request", () => { expect(gcpLogFilter(false)).toBe( - 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC"', + 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update" AND NOT logName:"serialconsole.googleapis.com"', ) expect(gcpLogFilter(true)).toBe(`${gcpLogFilter(false)} AND NOT resource.type="k8s_container"`) }) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 02a75bb433..b1548af7d9 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -17,11 +17,15 @@ const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` const DOCS_URL = "https://maple.dev/docs/integrations/gcp" -// Data Access audit logs record every API read, and load balancer health-check probes hit each -// backend every few seconds: both are high volume and say little about the workload. +// High volume that says little about a workload. Data Access audit logs record every API read. +// Load balancer health-check probes hit each backend every few seconds. A GKE cluster renews +// its leader-election leases all day, about 570 audit entries a minute for one idle node. A VM +// writes its serial console at boot, thousands of lines of raw terminal output. const DEFAULT_LOG_FILTER = [ 'NOT log_id("cloudaudit.googleapis.com/data_access")', 'NOT httpRequest.userAgent:"GoogleHC"', + 'NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update"', + 'NOT logName:"serialconsole.googleapis.com"', ] export const gcpLogFilter = (excludeGkeContainerLogs: boolean): string => @@ -202,7 +206,18 @@ unbind() { } ` +// Signed in as a person in Cloud Shell, gcloud's calls are counted against Google's own project, +// which has the API. With it off for a service account, every describe and role grant fails. +const RESOURCE_MANAGER = `# gcloud needs the Cloud Resource Manager API of the project its calls are counted against. For a +# service account that is its own project, usually this one. Most people do not need this, and +# not everyone may switch an API on, so a refusal here is not an error. +[ -z "$(off cloudresourcemanager.googleapis.com)" ] || + gcloud services enable cloudresourcemanager.googleapis.com --project="$PROJECT_ID" >/dev/null 2>&1 || true +` + interface AccessChecks { + /** Setup only: makes sure gcloud can read the project before the checks that read it. */ + readonly resourceManager: boolean /** The scope's name must be readable: true when a role the run needs includes reading it. */ readonly scopeMustOpen: boolean readonly billing: boolean @@ -251,7 +266,7 @@ allowed() { ` const access = (scopeType: GcpScopeType, checks: AccessChecks): string => ` -# ---- Checking access: nothing is changed before all of it passes ---- +# ---- Checking access: nothing is created or removed before all of it passes ---- section "Checking access" command -v gcloud >/dev/null 2>&1 || stop "gcloud is not installed here." "Run this script in Cloud Shell: https://shell.cloud.google.com" @@ -261,7 +276,7 @@ if [ -z "$ACCOUNT" ] || [ -z "$TOKEN" ]; then stop "You are not signed in to Google Cloud." "In Cloud Shell, click Authorize when it asks. Elsewhere, run: gcloud auth login" fi ok "Signed in as $ACCOUNT" -${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then +${checks.resourceManager ? RESOURCE_MANAGER : ""}${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then stop "Can't open project $PROJECT_ID as $ACCOUNT." "Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). If it is right, this account has no access to the project." "$(gcloud projects describe "$PROJECT_ID" 2>&1 || true)" fi ok "${scopeType === "project" ? "Project" : "Host project"} $PROJECT_ID found ($PROJECT_NAME)" @@ -294,8 +309,8 @@ const permissionChecks = ( if (!logs && !metrics) return [] const scope = SCOPES[scopeType] const administrator = "or have an administrator run this script." + // Switching an API on is not asked about: it is only needed for an API that is off. const host = [ - "serviceusage.services.enable", "resourcemanager.projects.getIamPolicy", "resourcemanager.projects.setIamPolicy", ...(logs @@ -332,9 +347,10 @@ const logsSetup = (scopeType: GcpScopeType): string => ` section "Log forwarding" DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" -# APIs in the host project: Pub/Sub carries the log entries, Cloud Logging routes them. -run "APIs enabled (Pub/Sub, Cloud Logging)" "Couldn't enable the Pub/Sub and Cloud Logging APIs." "$OWNER" \\ - gcloud services enable pubsub.googleapis.com logging.googleapis.com --project="$PROJECT_ID" +# APIs in the host project: Pub/Sub carries the log entries, Cloud Logging routes them, and +# Cloud Resource Manager is what gcloud grants the roles below through. +apis "Pub/Sub, Cloud Logging, Cloud Resource Manager" \\ + pubsub.googleapis.com logging.googleapis.com cloudresourcemanager.googleapis.com if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then ok "Topic already exists" @@ -423,9 +439,10 @@ section "Metrics and resources" # cloudasset.googleapis.com Cloud Asset Inventory, to list resources # iam.googleapis.com to create the service account below # iamcredentials.googleapis.com short-lived tokens for that account; no key is ever created -run "APIs enabled (Cloud Monitoring, Cloud Asset, IAM, IAM Credentials)" "Couldn't enable the Cloud Monitoring, Cloud Asset and IAM APIs." "$OWNER" \\ - gcloud services enable monitoring.googleapis.com cloudasset.googleapis.com \\ - iam.googleapis.com iamcredentials.googleapis.com --project="$PROJECT_ID" +# cloudresourcemanager.googleapis.com what gcloud grants the roles below through +apis "Cloud Monitoring, Cloud Asset, IAM, IAM Credentials, Cloud Resource Manager" \\ + monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com \\ + iamcredentials.googleapis.com cloudresourcemanager.googleapis.com # A service account that an earlier run deleted (metrics switched off) fails this describe as # well and is created again. Google makes that a new account under the same name, with none of @@ -583,6 +600,12 @@ ${ logs ? ` # ---- Which logs are forwarded ---- +# LOG_FILTER leaves out what is high volume and says little about a workload: Data Access audit +# logs, load balancer health checks, Kubernetes lease renewals and VM serial console output${ + input.logFilter === "exclude_gke_container_logs" + ? ",\n# and GKE container logs, as chosen in Maple" + : "" + }. # keep: an existing sink keeps its filter, a new sink gets LOG_FILTER. # set: LOG_FILTER replaces the sink's filter. # Filter syntax: https://cloud.google.com/logging/docs/view/logging-query-language @@ -602,10 +625,35 @@ ${output("Setup did not finish. Nothing needs undoing: fix this and paste the sc # exists : a look that fails for any reason falls through to the create, # which reports for itself. exists() { "$@" >/dev/null 2>&1; } + +# off : the APIs of the list that are not switched on in the host project. All of them +# when the list of enabled APIs cannot be read. +off() { + local on api nl=$'\\n' + on="$(gcloud services list --enabled --project="$PROJECT_ID" --format='value(config.name)' 2>/dev/null || true)" + for api in "$@"; do + case "$nl$on$nl" in *"$nl$api$nl"*) ;; *) printf ' %s' "$api" ;; esac + done +} + +# apis : switches on what is off. Asking first lets someone who may not +# switch APIs on run a script whose APIs are on already. +apis() { + local names="$1" missing + shift + missing="$(off "$@")" + if [ -n "$missing" ]; then + # Unquoted on purpose: one argument per API. + try "Couldn't switch on:$missing." "Service Usage Admin on project $PROJECT_ID" \\ + gcloud services enable $missing --project="$PROJECT_ID" + fi + ok "APIs enabled ($names)" +} ${logs && metrics === "on" ? "" : REMOVAL_HELPERS} printf 'Maple setup for %s\\n' "$SCOPE" printf ' Log forwarding %s\\n Metrics and resources %s\\n' ${sh(state(logs))} ${sh(metricsState)} ${access(scopeType, { + resourceManager: true, scopeMustOpen: metrics === "on", billing: metrics === "on", permissions: permissionChecks(scopeType, logs, metrics === "on"), @@ -624,7 +672,10 @@ else printf '\\nDone. Google Cloud is set up for Maple.\\n' fi if [ "$CONFIRMED" = 1 ]; then printf ' Maple confirms it within a minute: %s\\n' "$MAPLE_URL"; fi -if [ "$SINK_CREATED" = 1 ]; then printf ' Logs: a new sink takes up to 10 minutes to forward its first entries.\\n'; fi +if [ "$SINK_CREATED" = 1 ]; then + printf ' Logs: a new sink can take about 10 minutes to start forwarding. What is logged before\\n' + printf ' it does is not forwarded later.\\n' +fi if [ "$ACCOUNT_CREATED" = 1 ]; then printf ' Metrics: the first read lands within about 10 minutes.\\n'; fi `, ) @@ -650,7 +701,7 @@ ${output( "Cleanup did not finish. Fix this and paste the script again: it continues where it stopped.", )}${REMOVAL_HELPERS} printf 'Maple cleanup for %s\\n' "$SCOPE" -${access(input.scopeType, { scopeMustOpen: false, billing: false, permissions: [] })}${logsRemoval(input.scopeType, false)}${metricsRemoval(input.scopeType, false)}${ +${access(input.scopeType, { resourceManager: false, scopeMustOpen: false, billing: false, permissions: [] })}${logsRemoval(input.scopeType, false)}${metricsRemoval(input.scopeType, false)}${ input.pushEndpoint === undefined ? "" : ` diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts index 6acab4a9f1..5cef5c8f6f 100644 --- a/packages/domain/src/http/v2/integrations-gcp.ts +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -193,7 +193,7 @@ export type V2GcpUpdateConnectorRequest = Schema.Schema.Type Date: Fri, 9 Oct 2026 04:25:47 +0200 Subject: [PATCH 14/21] fix(integrations): Google Cloud script findings from a live run The access check now also covers the right to switch on an API that is off, tries the log filter out when the script is going to set it, and reads an account without any of the permissions as a wrong ID first. Removing a sink waits a minute before the topic goes, so Google does not log a routing error in the customer's project. Google's detail block is no longer printed under a failed step, three more of its refusals get their own remedy, a replaced filter is named in the closing line, and the wait for a new service account polls every 3 seconds. --- .../integrations/gcp/setup-scripts.test.ts | 112 ++++++++++++-- .../integrations/gcp/setup-scripts.ts | 140 ++++++++++++------ 2 files changed, 200 insertions(+), 52 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index 3103b9ff19..a250592d8b 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -166,7 +166,8 @@ done case "$url" in *testIamPermissions) echo "$url $header" >> "$CURL_LOG" - [ "$CURL_IAM" = ok ] || exit 22 + [ "$CURL_IAM" != unanswered ] || exit 22 + if [ "$CURL_IAM" = nothing ]; then echo "{}"; exit 0; fi lacking="\\"$CURL_LACKING\\"" echo "\${data//$lacking/}" ;; *) @@ -191,7 +192,10 @@ interface RunOptions { billing?: "True" | "False" /** The APIs that are on in the host project. None unless the test says so. */ apis?: ReadonlyArray - /** A permission the account does not hold, or `unanswered` when Google cannot be asked. */ + /** + * A permission the account does not hold, `everything` for an account without any, or + * `unanswered` when Google cannot be asked. + */ lacking?: string report?: "ok" | "unreachable" /** Extra arguments for the shell the text is fed to, and text typed after it. */ @@ -226,7 +230,12 @@ const run = (script: string, options: RunOptions = {}) => { GCLOUD_BILLING: options.billing ?? "True", GCLOUD_APIS: (options.apis ?? []).join("\n"), CURL_LOG: join(dir, "curl.log"), - CURL_IAM: options.lacking === "unanswered" ? "unanswered" : "ok", + CURL_IAM: + options.lacking === "unanswered" + ? "unanswered" + : options.lacking === "everything" + ? "nothing" + : "ok", CURL_LACKING: options.lacking ?? "", CURL_REPORT: options.report ?? "ok", }, @@ -251,7 +260,7 @@ const RESOURCE_MANAGER_ON = "services enable cloudresourcemanager.googleapis.com const isAccessCheck = (command: string) => command === API_LOOKUP || command === RESOURCE_MANAGER_ON || - /^(config get-value|auth print-access-token|projects describe|organizations describe|resource-manager folders describe|billing projects describe)/.test( + /^(config get-value|auth print-access-token|projects describe|organizations describe|resource-manager folders describe|billing projects describe|logging read)/.test( command, ) @@ -268,12 +277,13 @@ const publish = (members: string) => ) const logsSetupCommands = (scope: Scope) => [ - "services enable pubsub.googleapis.com logging.googleapis.com cloudresourcemanager.googleapis.com --project=acme-host", + // The last access check: whether the sink exists decides whether the filter is tried out. + `logging sinks describe ${NAME} ${scope.sink}`, + "services enable pubsub.googleapis.com logging.googleapis.com --project=acme-host", `pubsub topics describe ${NAME} --project=acme-host`, `pubsub topics create ${NAME} --project=acme-host`, `pubsub subscriptions describe ${NAME} --project=acme-host`, `pubsub subscriptions create ${NAME} --topic=${NAME} --project=acme-host --push-endpoint=${PUSH_ENDPOINT} --push-no-wrapper --ack-deadline=30 --message-retention-duration=1d --expiration-period=never --min-retry-delay=10s --max-retry-delay=600s`, - `logging sinks describe ${NAME} ${scope.sink}`, `logging sinks create ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(false)} ${scope.sinkWrite}`, `logging sinks describe ${NAME} ${scope.sink} --format=value(writerIdentity)`, `projects add-iam-policy-binding acme-host --member=${WRITER} --role=roles/logging.logWriter --condition=None`, @@ -282,7 +292,7 @@ const logsSetupCommands = (scope: Scope) => [ ] const metricsSetupCommands = (scope: Scope) => [ - "services enable monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com iamcredentials.googleapis.com cloudresourcemanager.googleapis.com --project=acme-host", + "services enable monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com iamcredentials.googleapis.com --project=acme-host", `iam service-accounts describe ${ACCOUNT} --project=acme-host`, `iam service-accounts create ${NAME} --project=acme-host --display-name=Maple metrics and resource reader`, `${scope.iam} add-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, @@ -338,9 +348,7 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { ...scope.open, "projects describe acme-host --format=value(name)", "billing projects describe acme-host --format=value(billingEnabled)", - // Each section asks which of its APIs are off. - API_LOOKUP, - API_LOOKUP, + `logging read ${gcpLogFilter(false)} --limit=1 --freshness=1h ${scope.sink}`, ]) expect(stdout).toBe(`Maple setup for ${scopeType} ${scope.id} Log forwarding on @@ -351,6 +359,7 @@ Checking access ${[...scope.found, ` ✓ ${host} acme-host found (Acme Production)`].join("\n")} ✓ Billing is enabled ✓ jane@acme.com has the permissions this script needs + ✓ Log filter accepted Log forwarding ✓ APIs enabled (Pub/Sub, Cloud Logging, Cloud Resource Manager) @@ -427,6 +436,7 @@ Done. Google Cloud matches your Maple switches. expect(stdout).toContain(` Log forwarding ✓ Log sink deleted + … waiting a minute for Google to stop routing to the topic ✓ Push subscription deleted ✓ Topic deleted ✓ Maple notified @@ -449,6 +459,7 @@ ${[...scope.found, ` ✓ ${host} acme-host found (Acme Production)`].join("\n") Log forwarding ✓ Log sink deleted + … waiting a minute for Google to stop routing to the topic ✓ Push subscription deleted ✓ Topic deleted @@ -570,7 +581,10 @@ Done. Everything was already in place. ) const { stdout, commands } = run(filtered("exclude_gke_container_logs"), { describe: "found" }) + expect(stdout).toContain(" ✓ Log filter accepted\n") expect(stdout).toContain(" ✓ Log sink up to date (filter replaced)") + // The closing line says what this run changed. + expect(stdout).not.toContain("Done. Everything was already in place.") expect(commands).toContain( `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(true)} --project=acme-host`, ) @@ -600,6 +614,23 @@ ${UNFINISHED}`), ).toBe(true) }) + it("prints Google's error without its machine-readable details", () => { + const { stdout } = run(setup(setupInput("project", { logs: true, metrics: true })), { + fail: "topics create", + answer: [ + "WARNING: This command is using service account impersonation.", + "ERROR: (gcloud.pubsub.topics.create) Failed to create topic: User not authorized to perform this action.", + "- '@type': type.googleapis.com/google.rpc.ErrorInfo", + " metadata:", + " troubleshooter_url: https://console.cloud.google.com/iam-admin/troubleshooter", + ].join("\n"), + }) + expect(stdout).toContain( + "User not authorized to perform this action.\n\n What to do: jane@acme.com needs Owner on project acme-host.", + ) + expect(stdout).not.toContain("troubleshooter_url") + }) + it("reads Google's other refusals for what they are, and never prints the secret back", () => { const script = setup(setupInput("project", { logs: true, metrics: true })) const propagating = run(script, { @@ -729,6 +760,67 @@ ${UNFINISHED}`) `) }) + it("tries the log filter out before anything is created, when the script is going to set it", () => { + const unparseable = + "ERROR: (gcloud.logging.read) INVALID_ARGUMENT: Unparseable filter: syntax error at line 1, column 16, token ' '" + const script = setup(setupInput("project", { logs: true, metrics: false })) + const refused = run(script, { fail: "logging read", answer: unparseable }) + expect(refused.status).toBe(1) + expect(refused.commands).toEqual([`logging sinks describe ${NAME} --project=acme-host`]) + expect(refused.stdout).toContain(` ✗ Google does not accept the log filter. + + ${unparseable} + + What to do: Google does not accept the log filter. Correct the filter near the top of the script and paste it again. Filter syntax: https://cloud.google.com/logging/docs/view/logging-query-language +`) + + // Someone who may not read logs can still set the sink up. + const unreadable = run(script, { fail: "logging read" }) + expect(unreadable.status).toBe(0) + expect(unreadable.stdout).not.toContain("Log filter accepted") + + // An existing sink that keeps its filter has none to try out. + const kept = run(script, { describe: "found", fail: "logging read", answer: unparseable }) + expect(kept.status).toBe(0) + expect(kept.access).not.toContainEqual(expect.stringContaining("logging read")) + }) + + it("stops in the access check when an API is off and the account may not switch it on", () => { + const { status, stdout, commands } = run( + setup(setupInput("project", { logs: true, metrics: true })), + { + apis: [ + "pubsub.googleapis.com", + "logging.googleapis.com", + "cloudresourcemanager.googleapis.com", + ], + lacking: "serviceusage.services.enable", + }, + ) + expect(status).toBe(1) + expect(commands).toEqual([]) + expect(stdout) + .toContain(` ✗ jane@acme.com is missing permissions on project acme-host: serviceusage.services.enable. + + What to do: This script has to switch on: monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com iamcredentials.googleapis.com. Ask for Service Usage Admin there, or have an administrator switch them on. +`) + }) + + it("reads an account without any of the permissions as a wrong ID first", () => { + const { status, stdout, commands } = run( + setup(setupInput("project", { logs: true, metrics: true })), + { lacking: "everything" }, + ) + expect(status).toBe(1) + expect(commands).toEqual([]) + expect(stdout) + .toContain(` ✗ jane@acme.com has none of the permissions this script needs on project acme-host. + + What to do: Check the ID in Maple. If it is right, this account has no rights there. Ask for Owner there, or have an administrator run this script. +`) + expect(stdout).not.toContain("pubsub.topics.create") + }) + it("names a missing permission before the first change", () => { const script = setup(setupInput("organization", { logs: true, metrics: true })) const onHost = run(script, { lacking: "resourcemanager.projects.setIamPolicy" }) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index b1548af7d9..80d35d96ef 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -103,10 +103,11 @@ ok() { printf ' \\342\\234\\223 %s\\n' "$1"; } # stop [Google's answer] stop() { printf ' \\342\\234\\227 %s\\n' "$1" - # Google can quote the push endpoint back, and this output may be sent to support. + # Google can quote the push endpoint back, and this output may be sent to support. The error + # line says what was refused; the machine-readable details that follow it are left out. if [ -n "\${3-}" ]; then printf '\\n' - printf '%s\\n' "$3" | sed -e 's/secret=[^ )&"]*/secret=HIDDEN/g' -e 's/^/ /' + printf '%s\\n' "$3" | sed -e "/^- '@type':/,\\$d" -e 's/secret=[^ )&"]*/secret=HIDDEN/g' -e 's/^/ /' fi printf '\\n What to do: %s\\n' "$2" printf '\\n%s\\n' ${sh(unfinished)} @@ -123,10 +124,13 @@ fix() { # Google words these two as a denial too, so they are read before the missing role. *"has not been used"* | *"is disabled"* | *SERVICE_DISABLED*) printf '%s' "Google is still switching an API on, or the API is off. Wait a minute and paste the script again." ;; - *"ervice account"*"does not exist"*) + *"ervice account"*"does not exist"* | *"Unknown service account"*) printf '%s' "Google has not published the new service account yet. Wait a minute and paste the script again." ;; - *PERMISSION_DENIED* | *"does not have permission"* | *"denied on resource"*) + # Pub/Sub words a refusal as "User not authorized to perform this action". + *PERMISSION_DENIED* | *"does not have permission"* | *"denied on resource"* | *"not authorized"*) printf '%s' "$ACCOUNT needs $2." ;; + *"Unparseable filter"*) + printf '%s' "Google does not accept the log filter. Correct the filter near the top of the script and paste it again. Filter syntax: https://cloud.google.com/logging/docs/view/logging-query-language" ;; *) printf '%s' "Read Google's answer above. If it is unclear, send this output to support@maple.dev." ;; esac } @@ -206,18 +210,55 @@ unbind() { } ` +// APIs in the host project. Cloud Resource Manager is what gcloud reads projects and grants +// roles through. +const LOG_APIS = ["pubsub.googleapis.com", "logging.googleapis.com", "cloudresourcemanager.googleapis.com"] +const METRIC_APIS = [ + "monitoring.googleapis.com", + "cloudasset.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "cloudresourcemanager.googleapis.com", +] + // Signed in as a person in Cloud Shell, gcloud's calls are counted against Google's own project, -// which has the API. With it off for a service account, every describe and role grant fails. -const RESOURCE_MANAGER = `# gcloud needs the Cloud Resource Manager API of the project its calls are counted against. For a -# service account that is its own project, usually this one. Most people do not need this, and -# not everyone may switch an API on, so a refusal here is not an error. -[ -z "$(off cloudresourcemanager.googleapis.com)" ] || - gcloud services enable cloudresourcemanager.googleapis.com --project="$PROJECT_ID" >/dev/null 2>&1 || true +// which has the Cloud Resource Manager API. With it off for a service account, every describe +// and role grant fails. +const apisOff = ( + apis: ReadonlyArray, +) => `# Which of the APIs this script needs are off in the host project, asked once. +APIS_OFF="$(off ${apis.join(" ")})" +# gcloud needs the Cloud Resource Manager API of the project its calls are counted against. For a +# service account that is its own project, usually this one, so it is switched on before the +# first read. Not everyone may switch an API on: a refusal here is reported further down. +case "$APIS_OFF" in *cloudresourcemanager*) + if gcloud services enable cloudresourcemanager.googleapis.com --project="$PROJECT_ID" >/dev/null 2>&1; then + APIS_OFF="\${APIS_OFF/ cloudresourcemanager.googleapis.com/}" + fi ;; +esac +` + +const sinkAndFilter = (sink: string) => ` +# A filter Google would refuse is found here, before the topic and subscription exist: reading +# one entry makes Google parse it. Not being allowed to read logs is not an error. +SINK_EXISTS=0 +if exists gcloud logging sinks describe "$SINK" ${sink}; then SINK_EXISTS=1; fi +if [ "$SINK_EXISTS" = 0 ] || [ "$LOG_FILTER_MODE" = set ]; then + if FILTER_ANSWER="$(gcloud logging read "$LOG_FILTER" --limit=1 --freshness=1h ${sink} 2>&1 >/dev/null)"; then + ok "Log filter accepted" + else + case "$FILTER_ANSWER" in *"Unparseable filter"*) + stop "Google does not accept the log filter." "$(fix "$FILTER_ANSWER" "")" "$FILTER_ANSWER" ;; + esac + fi +fi ` interface AccessChecks { - /** Setup only: makes sure gcloud can read the project before the checks that read it. */ - readonly resourceManager: boolean + /** Setup only: the APIs the enabled sections need in the host project. */ + readonly apis: ReadonlyArray + /** Setup with log forwarding on: the flag that places the sink. */ + readonly sink: string | undefined /** The scope's name must be readable: true when a role the run needs includes reading it. */ readonly scopeMustOpen: boolean readonly billing: boolean @@ -249,7 +290,7 @@ const PERMISSION_CHECK = ` # is not a refusal: the steps below then report for themselves. CHECKED=1 allowed() { - local resource="$1" where="$2" ask="$3" answer permission lacking="" + local resource="$1" where="$2" ask="$3" answer permission lacking="" held=0 shift 3 if ! answer="$(curl -fsS --max-time 15 -X POST -H 'Content-Type: application/json' \\ -H @<(printf 'Authorization: Bearer %s\\n' "$TOKEN") \\ @@ -259,9 +300,14 @@ allowed() { return 0 fi for permission in "$@"; do - case "$answer" in *"\\"$permission\\""*) ;; *) lacking="$lacking, $permission" ;; esac + case "$answer" in *"\\"$permission\\""*) held=1 ;; *) lacking="$lacking, $permission" ;; esac done - [ -z "$lacking" ] || stop "$ACCOUNT is missing permissions on $where: \${lacking#, }." "$ask" + if [ -z "$lacking" ]; then return 0; fi + # None of several held: a wrong ID is likelier than a list of missing rights. + if [ "$held" = 0 ] && [ "$#" -gt 1 ]; then + stop "$ACCOUNT has none of the permissions this script needs on $where." "Check the ID in Maple. If it is right, this account has no rights there. $ask" + fi + stop "$ACCOUNT is missing permissions on $where: \${lacking#, }." "$ask" } ` @@ -276,7 +322,7 @@ if [ -z "$ACCOUNT" ] || [ -z "$TOKEN" ]; then stop "You are not signed in to Google Cloud." "In Cloud Shell, click Authorize when it asks. Elsewhere, run: gcloud auth login" fi ok "Signed in as $ACCOUNT" -${checks.resourceManager ? RESOURCE_MANAGER : ""}${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then +${checks.apis.length === 0 ? "" : apisOff(checks.apis)}${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then stop "Can't open project $PROJECT_ID as $ACCOUNT." "Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). If it is right, this account has no access to the project." "$(gcloud projects describe "$PROJECT_ID" 2>&1 || true)" fi ok "${scopeType === "project" ? "Project" : "Host project"} $PROJECT_ID found ($PROJECT_NAME)" @@ -296,9 +342,12 @@ esac ([resource, where, ask, permissions]) => `allowed "${resource}" "${where}" "${ask}" \\\n ${permissions.join(" ")}\n`, ) - .join("")}[ "$CHECKED" = 0 ] || ok "$ACCOUNT has the permissions this script needs" + .join("")}# Switching an API on takes a right of its own, asked about only when one is off. +[ -z "$APIS_OFF" ] || allowed "projects/$PROJECT_ID" "project $PROJECT_ID" "This script has to switch on:$APIS_OFF. Ask for Service Usage Admin there, or have an administrator switch them on." \\ + serviceusage.services.enable +[ "$CHECKED" = 0 ] || ok "$ACCOUNT has the permissions this script needs" ` -}` +}${checks.sink === undefined ? "" : sinkAndFilter(checks.sink)}` /** What the enabled sections need, asked of Google before the first change. */ const permissionChecks = ( @@ -309,7 +358,6 @@ const permissionChecks = ( if (!logs && !metrics) return [] const scope = SCOPES[scopeType] const administrator = "or have an administrator run this script." - // Switching an API on is not asked about: it is only needed for an API that is off. const host = [ "resourcemanager.projects.getIamPolicy", "resourcemanager.projects.setIamPolicy", @@ -349,8 +397,7 @@ DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" # APIs in the host project: Pub/Sub carries the log entries, Cloud Logging routes them, and # Cloud Resource Manager is what gcloud grants the roles below through. -apis "Pub/Sub, Cloud Logging, Cloud Resource Manager" \\ - pubsub.googleapis.com logging.googleapis.com cloudresourcemanager.googleapis.com +apis "Pub/Sub, Cloud Logging, Cloud Resource Manager" ${LOG_APIS.join(" ")} if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then ok "Topic already exists" @@ -375,10 +422,11 @@ else gcloud pubsub subscriptions create "$SUBSCRIPTION" --topic="$TOPIC" "\${SUBSCRIPTION_FLAGS[@]}" fi -if exists gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then +if [ "$SINK_EXISTS" = 1 ]; then if [ "$LOG_FILTER_MODE" = set ]; then run "Log sink up to date (filter replaced)" "Couldn't update the log sink." "$SINK_ROLE" \\ gcloud logging sinks update "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} + FILTER_REPLACED=1 else run "Log sink up to date (filter kept)" "Couldn't update the log sink." "$SINK_ROLE" \\ gcloud logging sinks update "$SINK" "$DESTINATION" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} @@ -406,14 +454,18 @@ notify "Check message sent to Maple through the topic" '"logs":true' const logsRemoval = (scopeType: GcpScopeType, notifies: boolean): string => ` # ---- Log forwarding: off. Remove what an earlier run created. ---- -# The sink goes first, so nothing routes into a topic that is gone. The Logs Writer grant on the -# host project stays: the sink wrote as the logging service agent of the ${scopeType}, an identity +# The sink goes first, so nothing routes into a topic that is gone. Google keeps routing for a +# while after a sink is deleted, and with the topic gone by then it logs an error in this project +# and may notify its contacts, so the topic waits a minute. The Logs Writer grant on the host +# project stays: the sink wrote as the logging service agent of the ${scopeType}, an identity # every other sink there shares. section "Log forwarding" if remains "the log sink" "$SINK_ROLE" gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then run "Log sink deleted" "Couldn't delete the log sink." "$SINK_ROLE" \\ gcloud logging sinks delete "$SINK" ${SCOPES[scopeType].sink} --quiet REMOVED=1 + printf ' \\342\\200\\246 waiting a minute for Google to stop routing to the topic\\n' + sleep 60 fi if remains "the push subscription" "$OWNER" gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then run "Push subscription deleted" "Couldn't delete the push subscription." "$OWNER" \\ @@ -441,8 +493,7 @@ section "Metrics and resources" # iamcredentials.googleapis.com short-lived tokens for that account; no key is ever created # cloudresourcemanager.googleapis.com what gcloud grants the roles below through apis "Cloud Monitoring, Cloud Asset, IAM, IAM Credentials, Cloud Resource Manager" \\ - monitoring.googleapis.com cloudasset.googleapis.com iam.googleapis.com \\ - iamcredentials.googleapis.com cloudresourcemanager.googleapis.com + ${METRIC_APIS.join(" ")} # A service account that an earlier run deleted (metrics switched off) fails this describe as # well and is created again. Google makes that a new account under the same name, with none of @@ -458,22 +509,21 @@ fi # grant # Google can take a minute to publish a new service account, and until then answers that it does -# not exist. Only that answer is waited out. +# not exist. Only that answer is waited out, for a minute at most. WAITED=0 grant() { - local failed="$1" role="$2" output attempt + local failed="$1" role="$2" output tries=0 shift 2 - for attempt in 1 2 3 4 5 6 7; do - if output="$("$@" 2>&1)"; then return 0; fi - case "$output" in *"does not exist"* | *NOT_FOUND*) ;; *) break ;; esac - if [ "$attempt" = 7 ]; then break; fi + while ! output="$("$@" 2>&1)"; do + tries=$((tries + 1)) + case "$output" in *"does not exist"* | *NOT_FOUND*) ;; *) tries=20 ;; esac + if [ "$tries" -ge 20 ]; then stop "$failed" "$(fix "$output" "$role")" "$output"; fi if [ "$WAITED" = 0 ]; then printf ' \\342\\200\\246 waiting for Google to publish the new service account\\n' WAITED=1 fi - sleep 10 + sleep 3 done - stop "$failed" "$(fix "$output" "$role")" "$output" } # Read-only roles on the ${scopeType}${scopeType === "project" ? "" : ", inherited by every project under it"}: @@ -619,6 +669,7 @@ LOG_FILTER=${sh(gcpLogFilter(input.logFilter === "exclude_gke_container_logs"))} ? [] : [`MAPLE_SERVICE_ACCOUNT=${sh(input.mapleServiceAccountEmail)}`]), ])}SINK_CREATED=0 +FILTER_REPLACED=0 ACCOUNT_CREATED=0 REMOVED=0 ${output("Setup did not finish. Nothing needs undoing: fix this and paste the script again.")}${notify(logs)} @@ -627,7 +678,8 @@ ${output("Setup did not finish. Nothing needs undoing: fix this and paste the sc exists() { "$@" >/dev/null 2>&1; } # off : the APIs of the list that are not switched on in the host project. All of them -# when the list of enabled APIs cannot be read. +# when the list of enabled APIs cannot be read. Asking first lets someone who may not switch APIs +# on run a script whose APIs are on already. off() { local on api nl=$'\\n' on="$(gcloud services list --enabled --project="$PROJECT_ID" --format='value(config.name)' 2>/dev/null || true)" @@ -636,12 +688,13 @@ off() { done } -# apis : switches on what is off. Asking first lets someone who may not -# switch APIs on run a script whose APIs are on already. +# apis : switches on the ones the access check found off. apis() { - local names="$1" missing + local names="$1" api missing="" shift - missing="$(off "$@")" + for api in "$@"; do + case " $APIS_OFF " in *" $api "*) missing="$missing $api" ;; esac + done if [ -n "$missing" ]; then # Unquoted on purpose: one argument per API. try "Couldn't switch on:$missing." "Service Usage Admin on project $PROJECT_ID" \\ @@ -653,7 +706,8 @@ ${logs && metrics === "on" ? "" : REMOVAL_HELPERS} printf 'Maple setup for %s\\n' "$SCOPE" printf ' Log forwarding %s\\n Metrics and resources %s\\n' ${sh(state(logs))} ${sh(metricsState)} ${access(scopeType, { - resourceManager: true, + apis: [...new Set([...(logs ? LOG_APIS : []), ...(metrics === "on" ? METRIC_APIS : [])])], + sink: logs ? SCOPES[scopeType].sink : undefined, scopeMustOpen: metrics === "on", billing: metrics === "on", permissions: permissionChecks(scopeType, logs, metrics === "on"), @@ -664,8 +718,10 @@ ${access(scopeType, { ? metricsRemoval(scopeType, true) : "" } -if [ "$SINK_CREATED$ACCOUNT_CREATED$REMOVED" = 000 ]; then +if [ "$SINK_CREATED$ACCOUNT_CREATED$REMOVED$FILTER_REPLACED" = 0000 ]; then printf '\\nDone. Everything was already in place.\\n' +elif [ "$SINK_CREATED$ACCOUNT_CREATED$REMOVED" = 000 ]; then + printf '\\nDone. The log sink has the filter of this script. Everything else was already in place.\\n' elif [ "$REMOVED" = 1 ]; then printf '\\nDone. Google Cloud matches your Maple switches.\\n' else @@ -701,7 +757,7 @@ ${output( "Cleanup did not finish. Fix this and paste the script again: it continues where it stopped.", )}${REMOVAL_HELPERS} printf 'Maple cleanup for %s\\n' "$SCOPE" -${access(input.scopeType, { resourceManager: false, scopeMustOpen: false, billing: false, permissions: [] })}${logsRemoval(input.scopeType, false)}${metricsRemoval(input.scopeType, false)}${ +${access(input.scopeType, { apis: [], sink: undefined, scopeMustOpen: false, billing: false, permissions: [] })}${logsRemoval(input.scopeType, false)}${metricsRemoval(input.scopeType, false)}${ input.pushEndpoint === undefined ? "" : ` From 7a15a75ce5f8cb17d0480dff6c13f250b8e706ab Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 11:22:45 +0200 Subject: [PATCH 15/21] fix(integrations): Google Cloud script wording from the final live run A wrong ID is fixed by removing the connection and connecting the right one. The closing line of a run that created nothing no longer claims nothing was missing. A project-scope script says the project, not the host project. --- .../integrations/gcp/setup-scripts.test.ts | 17 +++++-- .../integrations/gcp/setup-scripts.ts | 51 +++++++++++-------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index a250592d8b..0e469c4239 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -385,6 +385,13 @@ Done. Google Cloud is set up for Maple. `) }) + it("says host project only where the scope is not the project itself", () => { + for (const enabled of [true, false]) { + const script = setup(setupInput(scopeType, { logs: enabled, metrics: enabled })) + expect(/host project/i.test(script)).toBe(scopeType !== "project") + } + }) + it("with logs only, sets up logs and removes an earlier metrics setup", () => { const script = setup(setupInput(scopeType, { logs: true, metrics: false })) expect(script).not.toContain(MAPLE_ACCOUNT) @@ -565,7 +572,7 @@ Metrics and resources ✓ Maple allowed to read as that account ✓ Maple notified -Done. Everything was already in place. +Done. Everything is in place. Maple confirms it within a minute: ${MAPLE_URL} `) expect(stdout).not.toMatch(/Logs: {4}|Metrics: /) @@ -584,7 +591,7 @@ Done. Everything was already in place. expect(stdout).toContain(" ✓ Log filter accepted\n") expect(stdout).toContain(" ✓ Log sink up to date (filter replaced)") // The closing line says what this run changed. - expect(stdout).not.toContain("Done. Everything was already in place.") + expect(stdout).not.toContain("Done. Everything is in place.") expect(commands).toContain( `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(true)} --project=acme-host`, ) @@ -678,7 +685,7 @@ Checking access ERROR: (gcloud) PERMISSION_DENIED: Permission denied on resource (or it may not exist). - What to do: Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). If it is right, this account has no access to the project. + What to do: Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). If it is wrong, remove the connection in Maple and connect the right ID: a connection's ID can't be changed. If it is right, this account has no access to the project. ${UNFINISHED}`) }) @@ -689,7 +696,7 @@ ${UNFINISHED}`) expect(both.commands).toEqual([]) expect(both.stdout).toContain(" ✗ Can't open organization 123456789012 as jane@acme.com.") expect(both.stdout).toContain( - "What to do: Check the ID with: gcloud organizations list. If it is right, this account needs Organization Administrator there.", + "What to do: Check the ID with: gcloud organizations list. If it is wrong, remove the connection in Maple and connect the right ID: a connection's ID can't be changed. If it is right, this account needs Organization Administrator there.", ) // Logs Configuration Writer does not include reading the organization. @@ -816,7 +823,7 @@ ${UNFINISHED}`) expect(stdout) .toContain(` ✗ jane@acme.com has none of the permissions this script needs on project acme-host. - What to do: Check the ID in Maple. If it is right, this account has no rights there. Ask for Owner there, or have an administrator run this script. + What to do: Check the ID in Maple. If it is wrong, remove the connection in Maple and connect the right ID: a connection's ID can't be changed. If it is right, this account has no rights there. Ask for Owner there, or have an administrator run this script. `) expect(stdout).not.toContain("pubsub.topics.create") }) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 80d35d96ef..018ccd7636 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -17,6 +17,10 @@ const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` const DOCS_URL = "https://maple.dev/docs/integrations/gcp" +// A connection's IDs are fixed once it exists, so a wrong one is fixed by connecting again. +const WRONG_ID = + "If it is wrong, remove the connection in Maple and connect the right ID: a connection's ID can't be changed." + // High volume that says little about a workload. Data Access audit logs record every API read. // Load balancer health-check probes hit each backend every few seconds. A GKE cluster renews // its leader-election leases all day, about 570 audit entries a minute for one idle node. A VM @@ -67,6 +71,10 @@ const SCOPES = { }, } as const +// A project is its own host project, so its script says "the project". +const hostProject = (scopeType: GcpScopeType): string => + scopeType === "project" ? "the project" : "the host project" + // A folder or organization sink also routes the logs of every project underneath. const includeChildren = (scopeType: GcpScopeType): string => scopeType === "project" ? "" : " --include-children" @@ -75,7 +83,7 @@ const variables = (target: GcpScriptTarget, more: ReadonlyArray): string const names = gcpConnectorResourceNames(target.connectorId) return ` # ---- This connection ---- -# PROJECT_ID is the host project: it holds Maple's Pub/Sub topic, subscription and service account. +# PROJECT_ID ${target.scopeType === "project" ? "" : "is the host project: it "}holds Maple's Pub/Sub topic, subscription and service account. PROJECT_ID=${sh(target.projectId)} SCOPE_ID=${sh(target.scopeId)} TOPIC=${sh(names.topic)} @@ -183,10 +191,10 @@ notify() { // otherwise be left routing into a topic the next step deletes. const REMOVAL_HELPERS = ` # remains : whether a resource is still -# there. NOT_FOUND means it is not, and so does a disabled API for the host project's own -# resources: setup switches their APIs on before it creates them (a sink can exist without). -# Any other answer stops the script: the resource may still be there, and nothing that it -# depends on is removed before it. +# there. NOT_FOUND means it is not, and so does a disabled API for the topic, the subscription +# and the service account: setup switches their APIs on before it creates them (a sink can exist +# without). Any other answer stops the script: the resource may still be there, and nothing that +# it depends on is removed before it. remains() { local what="$1" role="$2" output shift 2 @@ -225,8 +233,9 @@ const METRIC_APIS = [ // which has the Cloud Resource Manager API. With it off for a service account, every describe // and role grant fails. const apisOff = ( + scopeType: GcpScopeType, apis: ReadonlyArray, -) => `# Which of the APIs this script needs are off in the host project, asked once. +) => `# Which of the APIs this script needs are off in ${hostProject(scopeType)}, asked once. APIS_OFF="$(off ${apis.join(" ")})" # gcloud needs the Cloud Resource Manager API of the project its calls are counted against. For a # service account that is its own project, usually this one, so it is switched on before the @@ -272,7 +281,7 @@ const openScope = (scopeType: Exclude, mustOpen: boolea const label = scopeType === "folder" ? "Folder" : "Organization" return mustOpen ? `if ! SCOPE_NAME="$(${describe} --format='value(displayName)' 2>/dev/null)"; then - stop "Can't open $SCOPE as $ACCOUNT." "Check the ID with: ${scope.list}. If it is right, this account needs ${scope.iamRole} there." "$(${describe} 2>&1 || true)" + stop "Can't open $SCOPE as $ACCOUNT." "Check the ID with: ${scope.list}. ${WRONG_ID} If it is right, this account needs ${scope.iamRole} there." "$(${describe} 2>&1 || true)" fi ok "${label} $SCOPE_ID found ($SCOPE_NAME)" ` @@ -305,7 +314,7 @@ allowed() { if [ -z "$lacking" ]; then return 0; fi # None of several held: a wrong ID is likelier than a list of missing rights. if [ "$held" = 0 ] && [ "$#" -gt 1 ]; then - stop "$ACCOUNT has none of the permissions this script needs on $where." "Check the ID in Maple. If it is right, this account has no rights there. $ask" + stop "$ACCOUNT has none of the permissions this script needs on $where." "Check the ID in Maple. ${WRONG_ID} If it is right, this account has no rights there. $ask" fi stop "$ACCOUNT is missing permissions on $where: \${lacking#, }." "$ask" } @@ -322,8 +331,8 @@ if [ -z "$ACCOUNT" ] || [ -z "$TOKEN" ]; then stop "You are not signed in to Google Cloud." "In Cloud Shell, click Authorize when it asks. Elsewhere, run: gcloud auth login" fi ok "Signed in as $ACCOUNT" -${checks.apis.length === 0 ? "" : apisOff(checks.apis)}${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then - stop "Can't open project $PROJECT_ID as $ACCOUNT." "Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). If it is right, this account has no access to the project." "$(gcloud projects describe "$PROJECT_ID" 2>&1 || true)" +${checks.apis.length === 0 ? "" : apisOff(scopeType, checks.apis)}${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then + stop "Can't open project $PROJECT_ID as $ACCOUNT." "Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). ${WRONG_ID} If it is right, this account has no access to the project." "$(gcloud projects describe "$PROJECT_ID" 2>&1 || true)" fi ok "${scopeType === "project" ? "Project" : "Host project"} $PROJECT_ID found ($PROJECT_NAME)" ${ @@ -395,8 +404,8 @@ const logsSetup = (scopeType: GcpScopeType): string => ` section "Log forwarding" DESTINATION="pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC" -# APIs in the host project: Pub/Sub carries the log entries, Cloud Logging routes them, and -# Cloud Resource Manager is what gcloud grants the roles below through. +# APIs in ${hostProject(scopeType)}: Pub/Sub carries the log entries, Cloud Logging routes them, +# and Cloud Resource Manager is what gcloud grants the roles below through. apis "Pub/Sub, Cloud Logging, Cloud Resource Manager" ${LOG_APIS.join(" ")} if exists gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then @@ -456,9 +465,9 @@ const logsRemoval = (scopeType: GcpScopeType, notifies: boolean): string => ` # ---- Log forwarding: off. Remove what an earlier run created. ---- # The sink goes first, so nothing routes into a topic that is gone. Google keeps routing for a # while after a sink is deleted, and with the topic gone by then it logs an error in this project -# and may notify its contacts, so the topic waits a minute. The Logs Writer grant on the host -# project stays: the sink wrote as the logging service agent of the ${scopeType}, an identity -# every other sink there shares. +# and may notify its contacts, so the topic waits a minute. The Logs Writer grant on +# ${hostProject(scopeType)} stays: the sink wrote as the logging service agent of the ${scopeType}, +# an identity every other sink there shares. section "Log forwarding" if remains "the log sink" "$SINK_ROLE" gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then run "Log sink deleted" "Couldn't delete the log sink." "$SINK_ROLE" \\ @@ -486,7 +495,7 @@ const metricsSetup = (scopeType: GcpScopeType): string => ` # ---- Metrics and resources: on ---- section "Metrics and resources" -# APIs in the host project, the project Maple's reads are made through: +# APIs in ${scopeType === "project" ? "the project" : "the host project, the project Maple's reads are made through"}: # monitoring.googleapis.com Cloud Monitoring, to read metrics # cloudasset.googleapis.com Cloud Asset Inventory, to list resources # iam.googleapis.com to create the service account below @@ -677,9 +686,9 @@ ${output("Setup did not finish. Nothing needs undoing: fix this and paste the sc # which reports for itself. exists() { "$@" >/dev/null 2>&1; } -# off : the APIs of the list that are not switched on in the host project. All of them -# when the list of enabled APIs cannot be read. Asking first lets someone who may not switch APIs -# on run a script whose APIs are on already. +# off : the APIs of the list that are not switched on in ${hostProject(scopeType)}. All of +# them when the list of enabled APIs cannot be read. Asking first lets someone who may not switch +# APIs on run a script whose APIs are on already. off() { local on api nl=$'\\n' on="$(gcloud services list --enabled --project="$PROJECT_ID" --format='value(config.name)' 2>/dev/null || true)" @@ -719,9 +728,9 @@ ${access(scopeType, { : "" } if [ "$SINK_CREATED$ACCOUNT_CREATED$REMOVED$FILTER_REPLACED" = 0000 ]; then - printf '\\nDone. Everything was already in place.\\n' + printf '\\nDone. Everything is in place.\\n' elif [ "$SINK_CREATED$ACCOUNT_CREATED$REMOVED" = 000 ]; then - printf '\\nDone. The log sink has the filter of this script. Everything else was already in place.\\n' + printf '\\nDone. The log sink has the filter of this script. Everything else is in place.\\n' elif [ "$REMOVED" = 1 ]; then printf '\\nDone. Google Cloud matches your Maple switches.\\n' else From 570a38d707cbbd830a4f4b8a808de9ce7dace42d Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 16:59:10 +0200 Subject: [PATCH 16/21] feat(integrations): leave GKE container logs out of the default Google Cloud log filter Workloads that send their logs through OpenTelemetry would have every container line stored twice, and Google's copy carries no trace link. The default filter now excludes resource.type="k8s_container"; log_filter gains include_gke_container_logs and loses exclude_gke_container_logs. The script's comment above LOG_FILTER says what is left out, why, and how to include it. --- .../routes/v2/integrations-gcp.http.test.ts | 8 +++-- docs/api-v2.md | 2 +- .../integrations/GcpConnectorService.test.ts | 4 +-- .../integrations/gcp/setup-scripts.test.ts | 20 +++++++++---- .../integrations/gcp/setup-scripts.ts | 30 ++++++++++++------- .../domain/src/http/v2/integrations-gcp.ts | 4 +-- packages/primitives/src/index.ts | 4 +-- 7 files changed, 46 insertions(+), 26 deletions(-) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index 42228a0957..a8ce56f750 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -209,7 +209,7 @@ describe("v2 gcp integration over HTTP", () => { const path = `${CONNECTORS}/${created.body.id}` const both = await harness.request("POST", `${path}/setup_scripts`, admin.secret, { - log_filter: "exclude_gke_container_logs", + log_filter: "include_gke_container_logs", }) expect(both.status).toBe(200) expect(Object.keys(both.body).sort()).toEqual(["cleanup_script", "object", "setup_script"]) @@ -221,7 +221,7 @@ describe("v2 gcp integration over HTTP", () => { expect(both.body.setup_script).toContain("SCOPE_ID='123456789012'") expect(both.body.setup_script).toContain('--folder="$SCOPE_ID" --include-children') expect(both.body.setup_script).toContain( - `LOG_FILTER_MODE='set'\nLOG_FILTER='NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update" AND NOT logName:"serialconsole.googleapis.com" AND NOT resource.type="k8s_container"'`, + `LOG_FILTER_MODE='set'\nLOG_FILTER='NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update" AND NOT logName:"serialconsole.googleapis.com"'`, ) expect(both.body.setup_script).toContain( 'gcloud resource-manager folders add-iam-policy-binding "$SCOPE_ID"', @@ -232,9 +232,11 @@ describe("v2 gcp integration over HTTP", () => { // The cleanup script tells Maple that it ran, with the connector's secret. expect(both.body.cleanup_script).toContain("?secret=maple_gcp_") - // Without an option an existing sink keeps its filter; with logs off there is none to carry. + // Without an option an existing sink keeps its filter and a new one gets the default, which + // leaves out GKE container logs; with logs off there is none to carry. const kept = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) expect(kept.body.setup_script).toContain("LOG_FILTER_MODE='keep'") + expect(kept.body.setup_script).toContain(` AND NOT resource.type="k8s_container"'\n`) const unknown = await harness.request("POST", `${path}/setup_scripts`, admin.secret, { log_filter: "everything", }) diff --git a/docs/api-v2.md b/docs/api-v2.md index 0a1579affd..8621517d19 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -25,7 +25,7 @@ Within that group the role requirement is not uniform. Install, settings, unlink PlanetScale was the first promotion under that rule, and it shows what "a public dependency" means in practice: infrastructure-as-code setups drive `POST /v2/integrations/planetscale/metrics_token`, and a scripted caller needs a scoped API key, which only v2 has. That is the one step the OAuth flow cannot cover, because PlanetScale's metrics endpoints authenticate with service tokens instead of OAuth bearers. Once one operation of a provider is public the rest follows: splitting one provider across two tiers costs more than it buys, so the whole surface moved (status, connect, organization binding, disconnect, inventory, webhook config, query insights, events). The role split matches the chat group's: `status`, `databases`, `query_insights`, and `events` are ungated because the dashboard and service map render them for every member; everything that writes, plus `organizations` and `webhook_config` (it carries the signing secret), requires org-admin. -Google Cloud started on v2 for the same reason: there is no OAuth, so the whole setup is scriptable. A caller creates a connector for a project, a folder or an organization with a scoped key, chooses whether it forwards logs and whether Maple may read metrics and resources, and runs the returned `gcloud` script. The script tells Maple what it applied, which `status` returns as `applied_logs_enabled` and `applied_metrics_enabled`, so a caller can see within seconds that the run finished. `status` is ungated; creating, updating and deleting connectors and `setup_scripts` (it carries the push secret, and is a POST so a read-scoped key cannot fetch it) require org-admin. +Google Cloud started on v2 for the same reason: there is no OAuth, so the whole setup is scriptable. A caller creates a connector for a project, a folder or an organization with a scoped key, chooses whether it forwards logs and whether Maple may read metrics and resources, and runs the returned `gcloud` script. The script tells Maple what it applied, which `status` returns as `applied_logs_enabled` and `applied_metrics_enabled`, so a caller can see within seconds that the run finished. `setup_scripts` takes `log_filter`: `keep` (the default) leaves an existing sink's filter alone, `default` sets Maple's filter, which leaves out GKE container logs because instrumented workloads already send theirs over OTLP, and `include_gke_container_logs` forwards them too. `status` is ungated; creating, updating and deleting connectors and `setup_scripts` (it carries the push secret, and is a POST so a read-scoped key cannot fetch it) require org-admin. Cloudflare, GitHub, Hazel, and Railway live on the internal `integrations` group (`/internal/integrations`) with no v2 counterpart. They predate v2 and nothing public depends on them. Per the rule above, each gets promoted individually if something does. Scope families are derived mechanically from the first path segment under `/v2`, so every provider mounted at `/v2/integrations/` shares the `integrations:read` / `integrations:write` family. diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts index a1dd5610d1..4332167aa5 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.test.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -161,10 +161,10 @@ describe("GcpConnectorService", () => { assert.isTrue((yield* gcp.status(orgId)).metricsAvailable) const connector = yield* gcp.create(orgId, userId, organizationScope) const both = yield* gcp.scripts(orgId, connector.id, { - logFilter: "exclude_gke_container_logs", + logFilter: "include_gke_container_logs", }) assert.include(both.setupScript, `MAPLE_SERVICE_ACCOUNT='${MAPLE_ACCOUNT}'`) - assert.include(both.setupScript, 'NOT resource.type="k8s_container"') + assert.notInclude(both.setupScript, "k8s_container") assert.include(both.setupScript, '--organization="$SCOPE_ID" --include-children') assert.include(both.setupScript, "PROJECT_ID='acme-host'") assert.include(both.setupScript, "MAPLE_URL='http://127.0.0.1:3471/integrations?integration=gcp'") diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index 0e469c4239..f037a1608d 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -583,11 +583,21 @@ Done. Everything is in place. setup({ ...setupInput("project", { logs: true, metrics: false }), logFilter }) expect(filtered("keep")).toContain(`LOG_FILTER_MODE='keep'\nLOG_FILTER='${gcpLogFilter(false)}'`) expect(filtered("default")).toContain(`LOG_FILTER_MODE='set'\nLOG_FILTER='${gcpLogFilter(false)}'`) - expect(filtered("exclude_gke_container_logs")).toContain( + expect(filtered("include_gke_container_logs")).toContain( `LOG_FILTER_MODE='set'\nLOG_FILTER='${gcpLogFilter(true)}'`, ) + // The comment above the filter says which way GKE container logs went, and where to read why. + expect(filtered("default")).toContain( + "# It also leaves out GKE container logs: workloads that send their logs through OpenTelemetry\n", + ) + expect(filtered("include_gke_container_logs")).toContain( + "# GKE container logs are included, as chosen in Maple.", + ) + expect(filtered("include_gke_container_logs")).toContain( + "https://maple.dev/docs/integrations/gcp-opentelemetry\n", + ) - const { stdout, commands } = run(filtered("exclude_gke_container_logs"), { describe: "found" }) + const { stdout, commands } = run(filtered("include_gke_container_logs"), { describe: "found" }) expect(stdout).toContain(" ✓ Log filter accepted\n") expect(stdout).toContain(" ✓ Log sink up to date (filter replaced)") // The closing line says what this run changed. @@ -1048,11 +1058,11 @@ esac ) }) - it("leaves out high-volume noise by default, and GKE containers on request", () => { - expect(gcpLogFilter(false)).toBe( + it("leaves out high-volume noise and GKE container logs by default, and keeps the latter on request", () => { + expect(gcpLogFilter(true)).toBe( 'NOT log_id("cloudaudit.googleapis.com/data_access") AND NOT httpRequest.userAgent:"GoogleHC" AND NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update" AND NOT logName:"serialconsole.googleapis.com"', ) - expect(gcpLogFilter(true)).toBe(`${gcpLogFilter(false)} AND NOT resource.type="k8s_container"`) + expect(gcpLogFilter(false)).toBe(`${gcpLogFilter(true)} AND NOT resource.type="k8s_container"`) }) it("passes interpolated values to gcloud verbatim, whatever they contain", () => { diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 018ccd7636..62c3847827 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -16,6 +16,7 @@ import { Effect } from "effect" const sh = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` const DOCS_URL = "https://maple.dev/docs/integrations/gcp" +const OTEL_DOCS_URL = "https://maple.dev/docs/integrations/gcp-opentelemetry" // A connection's IDs are fixed once it exists, so a wrong one is fixed by connecting again. const WRONG_ID = @@ -25,17 +26,19 @@ const WRONG_ID = // Load balancer health-check probes hit each backend every few seconds. A GKE cluster renews // its leader-election leases all day, about 570 audit entries a minute for one idle node. A VM // writes its serial console at boot, thousands of lines of raw terminal output. -const DEFAULT_LOG_FILTER = [ +const NOISE = [ 'NOT log_id("cloudaudit.googleapis.com/data_access")', 'NOT httpRequest.userAgent:"GoogleHC"', 'NOT protoPayload.methodName="io.k8s.coordination.v1.leases.update"', 'NOT logName:"serialconsole.googleapis.com"', ] -export const gcpLogFilter = (excludeGkeContainerLogs: boolean): string => - [...DEFAULT_LOG_FILTER, ...(excludeGkeContainerLogs ? ['NOT resource.type="k8s_container"'] : [])].join( - " AND ", - ) +// Left out unless asked for: a workload that sends its logs to Maple through OpenTelemetry would +// have every container line stored twice, and Google's copy carries no trace link. +const GKE_CONTAINER_LOGS = 'NOT resource.type="k8s_container"' + +export const gcpLogFilter = (includeGkeContainerLogs: boolean): string => + [...NOISE, ...(includeGkeContainerLogs ? [] : [GKE_CONTAINER_LOGS])].join(" AND ") /** What a connector covers and where Maple's own resources live. */ export interface GcpScriptTarget { @@ -660,16 +663,21 @@ ${ ? ` # ---- Which logs are forwarded ---- # LOG_FILTER leaves out what is high volume and says little about a workload: Data Access audit -# logs, load balancer health checks, Kubernetes lease renewals and VM serial console output${ - input.logFilter === "exclude_gke_container_logs" - ? ",\n# and GKE container logs, as chosen in Maple" - : "" - }. +# logs, load balancer health checks, Kubernetes lease renewals and VM serial console output. +${ + input.logFilter === "include_gke_container_logs" + ? `# GKE container logs are included, as chosen in Maple. Workloads that send their logs through +# OpenTelemetry are then stored twice: ${OTEL_DOCS_URL}` + : `# It also leaves out GKE container logs: workloads that send their logs through OpenTelemetry +# would be stored twice. To include them, choose "Include GKE container logs" under Log filter +# in Maple, or delete the last clause of LOG_FILTER and set LOG_FILTER_MODE to set. +# Details: ${OTEL_DOCS_URL}` +} # keep: an existing sink keeps its filter, a new sink gets LOG_FILTER. # set: LOG_FILTER replaces the sink's filter. # Filter syntax: https://cloud.google.com/logging/docs/view/logging-query-language LOG_FILTER_MODE=${sh(input.logFilter === "keep" ? "keep" : "set")} -LOG_FILTER=${sh(gcpLogFilter(input.logFilter === "exclude_gke_container_logs"))} +LOG_FILTER=${sh(gcpLogFilter(input.logFilter === "include_gke_container_logs"))} ` : "" }${variables(input, [ diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts index 5cef5c8f6f..8927155f07 100644 --- a/packages/domain/src/http/v2/integrations-gcp.ts +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -193,14 +193,14 @@ export type V2GcpUpdateConnectorRequest = Schema.Schema.Type diff --git a/packages/primitives/src/index.ts b/packages/primitives/src/index.ts index 351f443f0c..4828090d0d 100644 --- a/packages/primitives/src/index.ts +++ b/packages/primitives/src/index.ts @@ -124,9 +124,9 @@ export type GcpScopeType = Schema.Schema.Type /** * The log filter a rendered setup script carries. `keep` leaves an existing sink's filter alone, - * the others replace it. + * the others replace it. `default` leaves out GKE container logs. */ -export const GcpLogFilter = Schema.Literals(["keep", "default", "exclude_gke_container_logs"]) +export const GcpLogFilter = Schema.Literals(["keep", "default", "include_gke_container_logs"]) export type GcpLogFilter = Schema.Schema.Type export const AlertDestinationId = MapleUuidId("@maple/AlertDestinationId", "Alert Destination ID") From 4277d81a8ba7642621b753c11d7b71d515fd6593 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 9 Oct 2026 17:30:18 +0200 Subject: [PATCH 17/21] fix(integrations): name the clause to delete in the Google Cloud script's filter comment The comment above LOG_FILTER names the GKE container clause instead of "the last clause", says that the logs are stored twice, and labels the keep and set lines with LOG_FILTER_MODE. --- .../integrations/gcp/setup-scripts.test.ts | 2 +- .../services/integrations/gcp/setup-scripts.ts | 18 +++++++++--------- .../domain/src/http/v2/integrations-gcp.ts | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index f037a1608d..5281527295 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -588,7 +588,7 @@ Done. Everything is in place. ) // The comment above the filter says which way GKE container logs went, and where to read why. expect(filtered("default")).toContain( - "# It also leaves out GKE container logs: workloads that send their logs through OpenTelemetry\n", + `# Log filter in Maple, or delete AND NOT resource.type="k8s_container" from LOG_FILTER and make\n`, ) expect(filtered("include_gke_container_logs")).toContain( "# GKE container logs are included, as chosen in Maple.", diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 62c3847827..7770c605ca 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -33,7 +33,7 @@ const NOISE = [ 'NOT logName:"serialconsole.googleapis.com"', ] -// Left out unless asked for: a workload that sends its logs to Maple through OpenTelemetry would +// Left out unless asked for: a workload that sends its logs to Maple over OpenTelemetry would // have every container line stored twice, and Google's copy carries no trace link. const GKE_CONTAINER_LOGS = 'NOT resource.type="k8s_container"' @@ -666,15 +666,15 @@ ${ # logs, load balancer health checks, Kubernetes lease renewals and VM serial console output. ${ input.logFilter === "include_gke_container_logs" - ? `# GKE container logs are included, as chosen in Maple. Workloads that send their logs through -# OpenTelemetry are then stored twice: ${OTEL_DOCS_URL}` - : `# It also leaves out GKE container logs: workloads that send their logs through OpenTelemetry -# would be stored twice. To include them, choose "Include GKE container logs" under Log filter -# in Maple, or delete the last clause of LOG_FILTER and set LOG_FILTER_MODE to set. -# Details: ${OTEL_DOCS_URL}` + ? `# GKE container logs are included, as chosen in Maple. Logs from workloads that also send them +# over OpenTelemetry are then stored twice: ${OTEL_DOCS_URL}` + : `# It also leaves out GKE container logs: logs from workloads that also send them over +# OpenTelemetry would be stored twice. To include them, choose "Include GKE container logs" under +# Log filter in Maple, or delete AND ${GKE_CONTAINER_LOGS} from LOG_FILTER and make +# LOG_FILTER_MODE 'set'. Details: ${OTEL_DOCS_URL}` } -# keep: an existing sink keeps its filter, a new sink gets LOG_FILTER. -# set: LOG_FILTER replaces the sink's filter. +# LOG_FILTER_MODE keep: an existing sink keeps its filter, a new sink gets LOG_FILTER. +# LOG_FILTER_MODE set: LOG_FILTER replaces the sink's filter. # Filter syntax: https://cloud.google.com/logging/docs/view/logging-query-language LOG_FILTER_MODE=${sh(input.logFilter === "keep" ? "keep" : "set")} LOG_FILTER=${sh(gcpLogFilter(input.logFilter === "include_gke_container_logs"))} diff --git a/packages/domain/src/http/v2/integrations-gcp.ts b/packages/domain/src/http/v2/integrations-gcp.ts index 8927155f07..818cbc5110 100644 --- a/packages/domain/src/http/v2/integrations-gcp.ts +++ b/packages/domain/src/http/v2/integrations-gcp.ts @@ -193,7 +193,7 @@ export type V2GcpUpdateConnectorRequest = Schema.Schema.Type Date: Sat, 10 Oct 2026 02:46:48 +0200 Subject: [PATCH 18/21] refactor(integrations): Google Cloud connector queries on effect-orm Same statements as before: the insert still resolves a taken scope with ON CONFLICT DO NOTHING on (org_id, scope_type, scope_id), and the update still reads the row FOR UPDATE inside its transaction. Timestamps are epoch milliseconds from the table, so the Date conversions are gone. --- .../integrations/GcpConnectorService.ts | 106 +++++++++--------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index 8b8d8d9d8c..a37609e342 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -1,5 +1,7 @@ import { randomBytes, randomUUID } from "node:crypto" -import { gcpConnectors, hashIngestKey, parseIngestKeyLookupHmacKey, type GcpConnectorRow } from "@maple/db" +import * as PG from "@maple-dev/effect-orm/postgres" +import { hashIngestKey, parseIngestKeyLookupHmacKey } from "@maple/db" +import { GcpConnectors, type GcpConnectorRow } from "@maple/db/tables" import { GcpMetricsUnavailableError, GcpScopeAlreadyConnectedError, @@ -16,13 +18,11 @@ import { type OrgId, type UserId, } from "@maple/domain/primitives" -import { and, asc, eq } from "drizzle-orm" import { Clock, Context, Effect, Layer, Option, Redacted } from "effect" import { decryptAes256Gcm, encryptAes256Gcm, parseBase64Aes256GcmKey } from "@maple/backend/platform/Crypto" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" import { Env } from "@maple/backend/platform/Env" -import { dateToMs, msToDate } from "@maple/backend/platform/time" import { renderGcpCleanupScript, renderGcpSetupScript } from "./gcp/setup-scripts" export interface GcpConnector { @@ -111,12 +111,12 @@ const toConnector = (row: GcpConnectorRow): GcpConnector => ({ projectId: row.projectId, logsEnabled: row.logsEnabled, metricsEnabled: row.metricsEnabled, - createdAt: dateToMs(row.createdAt), - lastLogReceivedAt: dateToMs(row.lastReceivedAt), + createdAt: row.createdAt, + lastLogReceivedAt: row.lastReceivedAt, lastLogError: row.lastError, appliedLogsEnabled: row.appliedLogsEnabled, appliedMetricsEnabled: row.appliedMetricsEnabled, - setupReportedAt: dateToMs(row.setupReportedAt), + setupReportedAt: row.setupReportedAt, }) const notFound = () => new IntegrationsNotFoundError({ message: "No such Google Cloud connector." }) @@ -164,11 +164,12 @@ export class GcpConnectorService extends Context.Service dbExecute((db) => - db - .select() - .from(gcpConnectors) - .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) - .limit(1), + db.run( + PG.from(GcpConnectors) + .select() + .where(($) => [$.orgId.eq(orgId), $.id.eq(connectorId)]) + .limit(1), + ), ).pipe( Effect.flatMap(([row]) => row === undefined ? Effect.fail(notFound()) : Effect.succeed(row), @@ -178,11 +179,12 @@ export class GcpConnectorService extends Context.Service - db - .select() - .from(gcpConnectors) - .where(eq(gcpConnectors.orgId, orgId)) - .orderBy(asc(gcpConnectors.createdAt), asc(gcpConnectors.id)), + db.run( + PG.from(GcpConnectors) + .select() + .where(($) => [$.orgId.eq(orgId)]) + .orderBy(["createdAt", "asc"], ["id", "asc"]), + ), ) return { metricsAvailable: mapleServiceAccountEmail !== undefined, @@ -210,26 +212,25 @@ export class GcpConnectorService extends Context.Service new IntegrationsPersistenceError({ message }), secretAad(id), ) - const now = msToDate(yield* Clock.currentTimeMillis) + const now = yield* Clock.currentTimeMillis const [row] = yield* dbExecute((db) => - db - .insert(gcpConnectors) - .values({ - id, - orgId, - ...input, - secretCiphertext: encrypted.ciphertext, - secretIv: encrypted.iv, - secretTag: encrypted.tag, - secretHash: hashIngestKey(secret, lookupHmacKey), - createdBy: userId, - createdAt: now, - updatedAt: now, - }) - .onConflictDoNothing({ - target: [gcpConnectors.orgId, gcpConnectors.scopeType, gcpConnectors.scopeId], - }) - .returning(), + db.run( + PG.insertInto(GcpConnectors) + .values({ + id, + orgId, + ...input, + secretCiphertext: encrypted.ciphertext, + secretIv: encrypted.iv, + secretTag: encrypted.tag, + secretHash: hashIngestKey(secret, lookupHmacKey), + createdBy: userId, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: ["orgId", "scopeType", "scopeId"] }) + .returning(), + ), ) if (row === undefined) { return yield* new GcpScopeAlreadyConnectedError({ @@ -250,17 +251,18 @@ export class GcpConnectorService extends Context.Service - db.transaction((tx) => + db.transaction( Effect.gen(function* () { // The row lock keeps one capability on when two updates race. - const [current] = yield* tx - .select() - .from(gcpConnectors) - .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) - .limit(1) - .for("update") + const [current] = yield* db.run( + PG.from(GcpConnectors) + .select() + .where(($) => [$.orgId.eq(orgId), $.id.eq(connectorId)]) + .limit(1) + .forUpdate(), + ) if (current === undefined) return undefined const flags = { logsEnabled: patch.logsEnabled ?? current.logsEnabled, @@ -276,10 +278,11 @@ export class GcpConnectorService extends Context.Service [$.id.eq(connectorId)]), + ) } return { row: { ...current, ...changes }, invalid } }), @@ -332,10 +335,11 @@ export class GcpConnectorService extends Context.Service - db - .delete(gcpConnectors) - .where(and(eq(gcpConnectors.orgId, orgId), eq(gcpConnectors.id, connectorId))) - .returning(), + db.run( + PG.deleteFrom(GcpConnectors) + .where(($) => [$.orgId.eq(orgId), $.id.eq(connectorId)]) + .returning(), + ), ) if (row === undefined) return yield* notFound() return { From 000b82f6beeaa6a0f06a14cbcd5bc356ca914307 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Wed, 14 Oct 2026 21:54:06 +0200 Subject: [PATCH 19/21] fix(gcp): follow main's ApiAuthorizationV2Live rename and effectful HMAC key parse --- apps/api/src/routes/v2/integrations-gcp.http.test.ts | 4 ++-- .../src/services/integrations/GcpConnectorService.ts | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index a8ce56f750..9ed78f878f 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -6,7 +6,7 @@ import { OrgId, UserId } from "@maple/domain/http" import { MapleApiV2, encodePublicId } from "@maple/domain/http/v2" import { Env } from "@maple/backend/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" -import { ApiAuthorizationV2Layer } from "@maple/backend/services/auth/ApiAuthorizationV2Layer" +import { ApiAuthorizationV2Live } from "@maple/backend/services/auth/ApiAuthorizationV2Live" import { AuthService } from "@maple/backend/services/auth/AuthService" import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" import { DashboardPersistenceService } from "@maple/backend/services/dashboards/DashboardPersistenceService" @@ -72,7 +72,7 @@ const makeHarness = (options: { readonly metricsAvailable: boolean } = { metrics Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(PlanetScaleServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), - Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiAuthorizationV2Live), Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), diff --git a/packages/backend/src/services/integrations/GcpConnectorService.ts b/packages/backend/src/services/integrations/GcpConnectorService.ts index a37609e342..faad3a8d8e 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.ts @@ -132,13 +132,10 @@ export class GcpConnectorService extends Context.Service new IntegrationsPersistenceError({ message }), ) // The ingest gateway hashes a pushed secret with this key and looks the row up by it. - const lookupHmacKey = yield* Effect.try({ - try: () => parseIngestKeyLookupHmacKey(Redacted.value(env.MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY)), - catch: () => - new IntegrationsPersistenceError({ - message: "MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY is required", - }), - }) + const lookupHmacKey = yield* parseIngestKeyLookupHmacKey( + Redacted.value(env.MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY), + (message) => new IntegrationsPersistenceError({ message }), + ) const ingestBaseUrl = env.MAPLE_INGEST_PUBLIC_URL.replace(/\/+$/, "") const mapleServiceAccountEmail = Option.getOrUndefined(env.MAPLE_GCP_SERVICE_ACCOUNT_EMAIL) const mapleUrl = `${env.MAPLE_APP_BASE_URL.replace(/\/+$/, "")}/integrations?integration=gcp` From ff85d4b2882679c0a137049548a7b47b6c9ad358 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Wed, 14 Oct 2026 23:23:18 +0200 Subject: [PATCH 20/21] fix(gcp): harden the setup and cleanup scripts for customer environments Security review of the pasted gcloud scripts. Secret and token exposure: - gcloud file logging off: it wrote every argument (the push secret) and the access token to ~/.config/gcloud/logs - zsh 5.3+ keeps the paste out of its history file (HISTORY_IGNORE), once - redaction also catches the secret JSON-escaped or URL-encoded - the paste refuses any value with a CR, TAB or other control character, which a terminal could read as the end of the here-document Re-runs and conflicts: - gcloud's calls are counted against the host project, not whichever project Cloud Shell has open (CLOUDSDK_CORE_PROJECT) - gcloud before 470 stops up front: its sink update lacks --include-children, so folder/org re-runs failed - a paused sink forwards again (--no-disabled) - a push subscription cut off from a deleted topic, or detached, is made anew instead of updated in place forever - IAM changes that race another policy change are tried again; removals use --all so a conditional binding of Maple's reader is not left - a resource gone between look and delete counts as removed - Ctrl-C ends between steps with a re-run hint Checks and messages: - the permission pre-check reads Google's 403 as a wrong folder or organization ID instead of skipping the check - the *.get permissions are checked, so an unreadable resource is named before the create that would collide with it - billing or permission checks that could not run say so - cleanup does not call a resource gone when its API is off - the impersonated service account is named as the one needing roles - domain-restricted sharing: allow Maple's one service account for this project, don't turn the policy off --- .../routes/v2/integrations-gcp.http.test.ts | 2 +- .../integrations/GcpConnectorService.test.ts | 2 +- .../integrations/gcp/setup-scripts.test.ts | 243 ++++++++++++++++-- .../integrations/gcp/setup-scripts.ts | 233 ++++++++++++----- 4 files changed, 398 insertions(+), 82 deletions(-) diff --git a/apps/api/src/routes/v2/integrations-gcp.http.test.ts b/apps/api/src/routes/v2/integrations-gcp.http.test.ts index 9ed78f878f..bf5c51a5de 100644 --- a/apps/api/src/routes/v2/integrations-gcp.http.test.ts +++ b/apps/api/src/routes/v2/integrations-gcp.http.test.ts @@ -188,7 +188,7 @@ describe("v2 gcp integration over HTTP", () => { ), }) // The connector is gone: its cleanup script has nothing to report to. - expect(deleted.body.cleanup_script).not.toContain("maple_gcp_") + expect(deleted.body.cleanup_script).not.toMatch(/maple_gcp_[A-Za-z0-9_-]{32}/) const gone = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {}) expect(gone.status).toBe(404) diff --git a/packages/backend/src/services/integrations/GcpConnectorService.test.ts b/packages/backend/src/services/integrations/GcpConnectorService.test.ts index 4332167aa5..1337c6353c 100644 --- a/packages/backend/src/services/integrations/GcpConnectorService.test.ts +++ b/packages/backend/src/services/integrations/GcpConnectorService.test.ts @@ -282,7 +282,7 @@ describe("GcpConnectorService", () => { const deleted = yield* gcp.delete(orgId, connector.id) assert.deepStrictEqual(deleted.connector, connector) assert.include(deleted.cleanupScript, gcpConnectorResourceNames(connector.id).sink) - assert.notInclude(deleted.cleanupScript, "maple_gcp_") + assert.notMatch(deleted.cleanupScript, /maple_gcp_[A-Za-z0-9_-]{32}/) assert.deepStrictEqual((yield* gcp.status(orgId)).connectors, []) const again = yield* Effect.flip(gcp.delete(orgId, connector.id)) assert.strictEqual(again._tag, "@maple/http/errors/IntegrationsNotFoundError") diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts index 5281527295..caf12e11c7 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.test.ts @@ -103,9 +103,19 @@ api_off() { echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2 exit 1 } +echo "$CLOUDSDK_CORE_PROJECT" >> "$GCLOUD_LOG.project" +echo "$CLOUDSDK_CORE_DISABLE_FILE_LOGGING" >> "$GCLOUD_LOG.filelogging" if [ -n "$GCLOUD_FAIL" ] && [[ "$*" == *"$GCLOUD_FAIL"* ]]; then denied; fi +# GCLOUD_RACE: the first command that contains it loses a race with another policy change. +if [ -n "$GCLOUD_RACE" ] && [[ "$*" == *"$GCLOUD_RACE"* ]] && [ ! -e "$GCLOUD_LOG.raced" ]; then + touch "$GCLOUD_LOG.raced" + echo "ERROR: (gcloud) ABORTED: There were concurrent policy changes. Please retry the whole read-modify-write with exponential backoff." >&2 + exit 1 +fi case "$*" in + "version"*) echo "$GCLOUD_VERSION" ;; "config get-value account") echo "$GCLOUD_ACCOUNT" ;; + "config get-value auth/impersonate_service_account") echo "$GCLOUD_IMPERSONATE" ;; "auth print-access-token") echo "ya29.token" ;; "projects describe"*) echo "Acme Production" ;; "organizations describe"* | "resource-manager folders describe"*) echo "acme.com" ;; @@ -119,6 +129,14 @@ case "$*" in denied) denied ;; api_off) api_off ;; esac ;; + *"subscriptions describe"*"--format=value(topic)") + case "$GCLOUD_DESCRIBE" in + found) echo "\${GCLOUD_TOPIC:-projects/acme-host/topics/${NAME}}" ;; + missing) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; + denied) denied ;; + api_off) api_off ;; + esac ;; + *"subscriptions describe"*"--format=value(detached)") echo "\${GCLOUD_DETACHED:-False}" ;; *" describe "*) case "$GCLOUD_DESCRIBE" in missing) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; @@ -166,10 +184,12 @@ done case "$url" in *testIamPermissions) echo "$url $header" >> "$CURL_LOG" - [ "$CURL_IAM" != unanswered ] || exit 22 - if [ "$CURL_IAM" = nothing ]; then echo "{}"; exit 0; fi + # With -w, the status follows the body on a line of its own. + [ "$CURL_IAM" != unanswered ] || exit 7 + if [ "$CURL_IAM" = unseen ]; then printf '{"error":{"status":"PERMISSION_DENIED"}}\n403'; exit 0; fi + if [ "$CURL_IAM" = nothing ]; then printf '{}\n200'; exit 0; fi lacking="\\"$CURL_LACKING\\"" - echo "\${data//$lacking/}" ;; + printf '%s\n200' "\${data//$lacking/}" ;; *) echo "$url $data" >> "$CURL_LOG" [ "$CURL_REPORT" = ok ] || exit 22 ;; @@ -193,11 +213,14 @@ interface RunOptions { /** The APIs that are on in the host project. None unless the test says so. */ apis?: ReadonlyArray /** - * A permission the account does not hold, `everything` for an account without any, or - * `unanswered` when Google cannot be asked. + * A permission the account does not hold, `everything` for an account without any, + * `unseen` for a resource it can't see (Google answers 403), or `unanswered` when Google + * cannot be asked. */ lacking?: string report?: "ok" | "unreachable" + /** Values the fake gcloud reads: GCLOUD_VERSION, GCLOUD_IMPERSONATE, GCLOUD_RACE, GCLOUD_TOPIC… */ + env?: Readonly> /** Extra arguments for the shell the text is fed to, and text typed after it. */ shell?: ReadonlyArray then?: string @@ -231,13 +254,14 @@ const run = (script: string, options: RunOptions = {}) => { GCLOUD_APIS: (options.apis ?? []).join("\n"), CURL_LOG: join(dir, "curl.log"), CURL_IAM: - options.lacking === "unanswered" - ? "unanswered" + options.lacking === "unanswered" || options.lacking === "unseen" + ? options.lacking : options.lacking === "everything" ? "nothing" : "ok", CURL_LACKING: options.lacking ?? "", CURL_REPORT: options.report ?? "ok", + ...options.env, }, }) return { @@ -249,6 +273,10 @@ const run = (script: string, options: RunOptions = {}) => { commands: read(log).filter((command) => !isAccessCheck(command)), access: read(log).filter(isAccessCheck), args: read(`${log}.args`), + /** The project each gcloud call was counted against. */ + projects: read(`${log}.project`), + /** Whether each gcloud call ran with its file log off. */ + fileLogging: read(`${log}.filelogging`), curl: read(join(dir, "curl.log")), } } @@ -260,7 +288,7 @@ const RESOURCE_MANAGER_ON = "services enable cloudresourcemanager.googleapis.com const isAccessCheck = (command: string) => command === API_LOOKUP || command === RESOURCE_MANAGER_ON || - /^(config get-value|auth print-access-token|projects describe|organizations describe|resource-manager folders describe|billing projects describe|logging read)/.test( + /^(version|config get-value|auth print-access-token|projects describe|organizations describe|resource-manager folders describe|billing projects describe|logging read)/.test( command, ) @@ -282,7 +310,7 @@ const logsSetupCommands = (scope: Scope) => [ "services enable pubsub.googleapis.com logging.googleapis.com --project=acme-host", `pubsub topics describe ${NAME} --project=acme-host`, `pubsub topics create ${NAME} --project=acme-host`, - `pubsub subscriptions describe ${NAME} --project=acme-host`, + `pubsub subscriptions describe ${NAME} --project=acme-host --format=value(topic)`, `pubsub subscriptions create ${NAME} --topic=${NAME} --project=acme-host --push-endpoint=${PUSH_ENDPOINT} --push-no-wrapper --ack-deadline=30 --message-retention-duration=1d --expiration-period=never --min-retry-delay=10s --max-retry-delay=600s`, `logging sinks create ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(false)} ${scope.sinkWrite}`, `logging sinks describe ${NAME} ${scope.sink} --format=value(writerIdentity)`, @@ -317,9 +345,9 @@ const isLookup = (command: string) => command.includes(" describe ") || command const metricsRemovalCommands = (scope: Scope) => [ ACCOUNT_LOOKUP, - `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --condition=None`, - `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --condition=None`, - `projects remove-iam-policy-binding acme-host --member=serviceAccount:${ACCOUNT} --role=roles/serviceusage.serviceUsageConsumer --condition=None`, + `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/monitoring.viewer --all`, + `${scope.iam} remove-iam-policy-binding ${scope.id} --member=serviceAccount:${ACCOUNT} --role=roles/cloudasset.viewer --all`, + `projects remove-iam-policy-binding acme-host --member=serviceAccount:${ACCOUNT} --role=roles/serviceusage.serviceUsageConsumer --all`, `iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`, ] @@ -341,8 +369,10 @@ describe.each(scopeTypes)("renderGcpSetupScript for a %s", (scopeType) => { publish('"metrics":true'), ]) expect(access).toEqual([ + 'version --format=value("Google Cloud SDK")', "config get-value account", "auth print-access-token", + "config get-value auth/impersonate_service_account", API_LOOKUP, RESOURCE_MANAGER_ON, ...scope.open, @@ -400,7 +430,7 @@ Done. Google Cloud is set up for Maple. // Everything exists already, so the log pipeline is updated in place, filter untouched. expect(commands.filter((command) => command.includes(" create "))).toEqual([]) expect(commands).toContain( - `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} ${scope.sinkWrite}`, + `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --no-disabled ${scope.sinkWrite}`, ) expect(commands).toContainEqual( expect.stringMatching(/^pubsub subscriptions update .*--push-no-wrapper/), @@ -493,7 +523,7 @@ describe("the text that is copied", () => { it("is the script as a here-document for a child bash", () => { const lines = script.split("\n") expect(lines.slice(0, 3)).toEqual([ - ` { [ -z "\${BASH_VERSION-}" ] || case "$(history 1)" in *MAPLE_SETUP_SCRIPT*) history -d -1 ;; esac 2>/dev/null`, + ` { [ -z "\${ZSH_VERSION-}" ] || case "\${HISTORY_IGNORE-}" in *"*MAPLE_SETUP_SCRIPT*"*) ;; *) HISTORY_IGNORE="(\${HISTORY_IGNORE:+$HISTORY_IGNORE|}*MAPLE_SETUP_SCRIPT*)" ;; esac; [ -z "\${BASH_VERSION-}" ] || case "$(history 1)" in *MAPLE_SETUP_SCRIPT*) history -d -1 ;; esac 2>/dev/null`, "bash /dev/fd/3 3<<'MAPLE_SETUP_SCRIPT'", "#!/usr/bin/env bash", ]) @@ -541,6 +571,17 @@ describe("the text that is copied", () => { ), ) expect(result._tag).toBe("Failure") + + // A terminal ends a pasted line at a CR, and a TAB is completion. + for (const hostileUrl of [ + "https://x\rMAPLE_SETUP_SCRIPT\rtouch pwned", + "https://x\n\tMAPLE_SETUP_SCRIPT", + ]) { + const pasted = Effect.runSync( + Effect.result(renderGcpSetupScript({ ...hostile, mapleUrl: hostileUrl })), + ) + expect(pasted._tag).toBe("Failure") + } }) }) @@ -603,7 +644,7 @@ Done. Everything is in place. // The closing line says what this run changed. expect(stdout).not.toContain("Done. Everything is in place.") expect(commands).toContain( - `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --log-filter=${gcpLogFilter(true)} --project=acme-host`, + `logging sinks update ${NAME} pubsub.googleapis.com/projects/acme-host/topics/${NAME} --no-disabled --log-filter=${gcpLogFilter(true)} --project=acme-host`, ) }) @@ -664,6 +705,14 @@ ${UNFINISHED}`), }) expect(quoted.stdout).toContain("?secret=HIDDEN).") expect(quoted.stdout).not.toContain(SECRET) + // However Google spells the endpoint back, escaped or encoded. + for (const spelled of [`secret\\u003d${SECRET}`, `secret%3D${SECRET}`, `'secret': '${SECRET}'`]) { + const answer = run(script, { + fail: "subscriptions create", + answer: `ERROR: (gcloud) INVALID_ARGUMENT: ${spelled}`, + }) + expect(answer.stdout).not.toContain(SECRET) + } expect(quoted.stdout).toContain( "What to do: Read Google's answer above. If it is unclear, send this output to support@maple.dev.", ) @@ -867,6 +916,80 @@ ${UNFINISHED}`) expect(unanswered.stdout).not.toContain("has the permissions") }) + it("reads a folder this account can't see as a wrong ID, before the first change", () => { + const { status, stdout, commands } = run( + setup(setupInput("folder", { logs: true, metrics: false })), + { + lacking: "unseen", + }, + ) + expect(status).toBe(1) + expect(commands).toEqual([]) + expect(stdout).toContain(` ✗ jane@acme.com can't see project acme-host. + + What to do: Check the ID in Maple.`) + }) + + it("says what it could not check instead of passing over it", () => { + const script = setup(setupInput("project", { logs: true, metrics: true })) + const { status, stdout } = run(script, { lacking: "unanswered", billing: "" as "True" }) + expect(status).toBe(0) + expect(stdout).toContain( + " • Couldn't tell whether billing is on. Cloud Monitoring only answers for projects with billing.\n", + ) + expect(stdout).toContain( + " • Couldn't ask Google which permissions jane@acme.com holds. A step it may not take says so.\n", + ) + }) + + it("doesn't call a resource gone when its API is off, in the cleanup", () => { + const hostOff = `#!/usr/bin/env bash +echo "$*" >> "$GCLOUD_LOG" +case "$*" in + "config get-value account") echo jane@acme.com ;; + "auth print-access-token") echo ya29.token ;; + "projects describe"*) echo "Acme Production" ;; + *"sinks describe"*) echo "ERROR: (gcloud) NOT_FOUND: Resource not found" >&2; exit 1 ;; + *) echo "ERROR: (gcloud) API has not been used in project 1 before or it is disabled" >&2; exit 1 ;; +esac +` + const { stdout } = run(cleanup(SCOPES.project), { gcloud: hostOff }) + expect(stdout).toContain( + " • Couldn't look for the read-only service account: its API is off in project acme-host. If a setup run made one, switch the API on and paste this again.\n", + ) + expect(stdout).toContain("\nDone, but not all of it could be checked: see above.\n") + expect(stdout).not.toContain("Everything the setup script created is gone") + + // A setup run with the API off has made nothing there, so it has nothing to say. + const setupRun = run(setup(setupInput("project", { logs: true, metrics: false })), { + gcloud: hostOff, + }) + expect(setupRun.stdout).not.toContain("its API is off") + }) + + it("takes a resource that is gone by the time it is deleted for removed", () => { + const { status, stdout } = run(cleanup(SCOPES.project), { + describe: "found", + fail: "topics delete", + answer: "ERROR: (gcloud.pubsub.topics.delete) NOT_FOUND: Resource not found (resource=maple).", + }) + expect(status).toBe(0) + expect(stdout).toContain(" ✓ Topic deleted\n") + }) + + it("ends between two steps on Ctrl-C and says a re-run picks up from there", () => { + const script = setup(setupInput("project", { logs: true, metrics: true })) + // The signal lands where a step has just finished. + const marker = '\nsection "Log forwarding"\n' + expect(script).toContain(marker) + const { status, stdout } = run(script.replace(marker, () => `\nkill -INT $$${marker}`)) + expect(status).toBe(130) + expect(stdout).toContain( + "\n\nSetup interrupted. Nothing needs undoing: paste the script again to finish it.\n", + ) + expect(stdout).not.toContain("Topic created") + }) + it("finishes when Maple cannot be reached, and does not claim a confirmation", () => { const { status, stdout } = run(setup(setupInput("project", { logs: false, metrics: true })), { report: "unreachable", @@ -1034,9 +1157,97 @@ esac ) }) + it("makes the push subscription anew when its topic was deleted or it was detached", () => { + const script = setup(setupInput("project", { logs: true, metrics: false })) + const recreated = [ + `pubsub subscriptions delete ${NAME} --project=acme-host --quiet`, + expect.stringMatching(`^pubsub subscriptions create ${NAME} --topic=${NAME} `), + ] + for (const env of [{ GCLOUD_TOPIC: "_deleted-topic_" }, { GCLOUD_DETACHED: "True" }]) { + const { status, stdout, commands } = run(script, { describe: "found", env }) + expect(status).toBe(0) + expect(commands.filter((command) => command.startsWith("pubsub subscriptions "))).toEqual([ + `pubsub subscriptions describe ${NAME} --project=acme-host --format=value(topic)`, + ...(env.GCLOUD_DETACHED === undefined + ? [] + : [`pubsub subscriptions describe ${NAME} --project=acme-host --format=value(detached)`]), + ...recreated, + ]) + expect(stdout).toContain( + " ✓ Push subscription cut off from the topic deleted\n ✓ Push subscription created\n", + ) + } + + // One still on the topic is updated in place. + const attached = run(script, { describe: "found" }) + expect(attached.commands).not.toContainEqual(expect.stringContaining("subscriptions delete")) + }) + + it("tries an IAM change again when it raced another change to the same policy", () => { + const granted = run(setup(setupInput("folder", { logs: true, metrics: true })), { + env: { GCLOUD_RACE: "add-iam-policy-binding" }, + }) + expect(granted.status).toBe(0) + expect( + granted.commands.filter((command) => command.includes("roles/logging.logWriter")), + ).toHaveLength(2) + expect(granted.stdout).not.toContain("waiting for Google") + + const removed = run(cleanup(SCOPES.folder), { + describe: "found", + env: { GCLOUD_RACE: "remove-iam-policy-binding" }, + }) + expect(removed.status).toBe(0) + expect( + removed.commands.filter((command) => command.includes("roles/monitoring.viewer")), + ).toHaveLength(2) + expect(removed.commands.at(-1)).toBe( + `iam service-accounts delete ${ACCOUNT} --project=acme-host --quiet`, + ) + }) + + it("stops before the first change on a gcloud whose sink update cannot re-run a folder setup", () => { + const script = setup(setupInput("folder", { logs: true, metrics: true })) + const old = run(script, { env: { GCLOUD_VERSION: "469.0.0" } }) + expect(old.status).toBe(1) + expect(old.commands).toEqual([]) + expect(old.stdout).toContain(" ✗ gcloud 469.0.0 is too old for this script.") + expect(run(script, { env: { GCLOUD_VERSION: "470.0.0" } }).status).toBe(0) + }) + + it("counts every gcloud call against the host project, whatever the caller's settings say", () => { + const { projects } = run(setup(setupInput("organization", { logs: true, metrics: true })), { + env: { CLOUDSDK_CORE_PROJECT: "unrelated-project" }, + }) + expect(new Set(projects)).toEqual(new Set(["acme-host"])) + }) + + it("keeps gcloud from writing its arguments, the secret among them, to its log files", () => { + for (const script of [ + setup(setupInput("project", { logs: true, metrics: true })), + cleanup(SCOPES.project, PUSH_ENDPOINT), + ]) { + const { fileLogging } = run(script, { describe: "found" }) + expect(fileLogging.length).toBeGreaterThan(0) + expect(new Set(fileLogging)).toEqual(new Set(["1"])) + } + }) + + it("names the impersonated service account as the one that needs the roles", () => { + const { stdout } = run(setup(setupInput("project", { logs: true, metrics: true })), { + fail: "sinks create", + env: { GCLOUD_IMPERSONATE: "deployer@acme-host.iam.gserviceaccount.com" }, + }) + expect(stdout).toContain(" ✓ Signed in as deployer@acme-host.iam.gserviceaccount.com\n") + expect(stdout).toContain( + "What to do: deployer@acme-host.iam.gserviceaccount.com needs Logs Configuration Writer on project acme-host.", + ) + }) + it("carries no secret and reports nothing once the connector is gone", () => { const script = cleanup(SCOPES.project) - expect(script).not.toContain("maple_gcp_") + expect(script).not.toMatch(/maple_gcp_[A-Za-z0-9_-]{20,}/) + expect(script).not.toContain("/v1/logpush/") expect(script).not.toContain("PUSH_ENDPOINT=") const { status, stdout, curl } = run(script, { describe: "found" }) expect(status).toBe(0) diff --git a/packages/backend/src/services/integrations/gcp/setup-scripts.ts b/packages/backend/src/services/integrations/gcp/setup-scripts.ts index 7770c605ca..97baeffba4 100644 --- a/packages/backend/src/services/integrations/gcp/setup-scripts.ts +++ b/packages/backend/src/services/integrations/gcp/setup-scripts.ts @@ -88,6 +88,9 @@ const variables = (target: GcpScriptTarget, more: ReadonlyArray): string # ---- This connection ---- # PROJECT_ID ${target.scopeType === "project" ? "" : "is the host project: it "}holds Maple's Pub/Sub topic, subscription and service account. PROJECT_ID=${sh(target.projectId)} +# gcloud counts a person's calls against its current project, in Cloud Shell whichever one the +# console has open. They belong to this one, whatever the caller's settings say. +export CLOUDSDK_CORE_PROJECT="$PROJECT_ID" SCOPE_ID=${sh(target.scopeId)} TOPIC=${sh(names.topic)} SUBSCRIPTION=${sh(names.subscription)} @@ -104,12 +107,16 @@ IAM_ROLE="${SCOPES[target.scopeType].iamRole} on $SCOPE" // Google's own output is shown only for a step that fails, under what failed and above what to // do about it. `stop` ends the script: every step can be run again, so nothing is rolled back. -const output = (unfinished: string) => ` +const output = (unfinished: string, interrupted: string) => ` # ---- What this script prints ---- # Its marks are spelled as bytes: the text you paste is plain ASCII, which every terminal and # locale passes on unchanged. section() { printf '\\n%s\\n' "$1"; } ok() { printf ' \\342\\234\\223 %s\\n' "$1"; } +note() { printf ' \\342\\200\\242 %s\\n' "$1"; } +# Ctrl-C ends the script between two steps, which a re-run picks up from. +interrupted() { printf '\\n\\n%s\\n' ${sh(interrupted)}; exit 130; } +trap interrupted INT # stop [Google's answer] stop() { @@ -118,7 +125,7 @@ stop() { # line says what was refused; the machine-readable details that follow it are left out. if [ -n "\${3-}" ]; then printf '\\n' - printf '%s\\n' "$3" | sed -e "/^- '@type':/,\\$d" -e 's/secret=[^ )&"]*/secret=HIDDEN/g' -e 's/^/ /' + printf '%s\\n' "$3" | sed -e "/^- '@type':/,\\$d" -e 's/secret=[^ )&"]*/secret=HIDDEN/g' -e 's/maple_gcp_[A-Za-z0-9_-]*/maple_gcp_HIDDEN/g' -e 's/^/ /' fi printf '\\n What to do: %s\\n' "$2" printf '\\n%s\\n' ${sh(unfinished)} @@ -131,7 +138,7 @@ fix() { *"requires billing"* | *BILLING_DISABLED* | *"Billing must be enabled"* | *BILLING_NOT_FOUND*) printf '%s' "Project $PROJECT_ID has no active billing account. Link one at https://console.cloud.google.com/billing/linkedaccount?project=$PROJECT_ID" ;; *allowedPolicyMember* | *"permitted customer"*) - printf '%s' "An organization policy (domain restricted sharing) blocks this grant. Ask an Organization Policy Administrator to lift it for project $PROJECT_ID while this script runs. See ${DOCS_URL}#domain-restricted-sharing" ;; + printf '%s' "An organization policy (domain restricted sharing) blocks granting Maple's service account \${MAPLE_SERVICE_ACCOUNT-} on project $PROJECT_ID. Ask an Organization Policy Administrator to allow that one service account, for this project only (iam.managed.allowedPolicyMembers). Don't turn the policy off. See ${DOCS_URL}#domain-restricted-sharing" ;; # Google words these two as a denial too, so they are read before the missing role. *"has not been used"* | *"is disabled"* | *SERVICE_DISABLED*) printf '%s' "Google is still switching an API on, or the API is off. Wait a minute and paste the script again." ;; @@ -161,6 +168,10 @@ run() { ok "$worked" } +# conflict : an IAM change raced another change to the same policy, which gcloud +# does not try again. Trying again is safe: each change is read afresh. +conflict() { case "$1" in *"concurrent policy changes"* | *ABORTED*) return 0 ;; *) return 1 ;; esac; } + # report : the message that tells Maple what a run did. It has # the form of a log entry; Maple reads it and does not store it. report() { @@ -184,7 +195,7 @@ notify() { } >/dev/null 2>&1; then ok "$1" elif [ "$CONFIRMED" = 1 ]; then - printf " \\342\\200\\242 Couldn't reach Maple to confirm. Maple shows the connection as pending until a later run reaches it.\\n" + note "Couldn't reach Maple to confirm. Maple shows the connection as pending until a later run reaches it." CONFIRMED=0 fi } @@ -205,19 +216,45 @@ remains() { case "$2:$output" in *NOT_FOUND*) return 1 ;; logging:*) ;; - *"has not been used"* | *"is disabled"*) return 1 ;; + *"has not been used"* | *"is disabled"*) api_off "$what"; return 1 ;; esac stop "Couldn't check whether $what still exists." "$(fix "$output" "$role")" "$output" } +# api_off : an API switched off after a setup run would hide what it made. The cleanup +# says so instead of calling it gone; a setup run with the API off has made nothing there. +UNCHECKED=0 +api_off() { + [ "\${ANY_API_OFF_MATTERS-0}" = 1 ] || return 0 + note "Couldn't look for $1: its API is off in project $PROJECT_ID. If a setup run made one, switch the API on and paste this again." + UNCHECKED=1 +} + +# remove : +# one that is gone already, a moment after it was seen, is fine. +remove() { + local worked="$1" failed="$2" role="$3" output + shift 3 + if ! output="$("$@" 2>&1)"; then + case "$output" in *NOT_FOUND*) ;; *) stop "$failed" "$(fix "$output" "$role")" "$output" ;; esac + fi + ok "$worked" + REMOVED=1 +} + # unbind : a binding that is # already gone is fine. unbind() { - local role="$1" output + local role="$1" output tries=0 shift - if output="$("$@" 2>&1)"; then return 0; fi - case "$output" in *"Policy binding"*"not found"*) return 0 ;; esac - stop "Couldn't remove the read-only roles." "$(fix "$output" "$role")" "$output" + until output="$("$@" 2>&1)"; do + case "$output" in *"Policy binding"*"not found"*) return 0 ;; esac + tries=$((tries + 1)) + if ! conflict "$output" || [ "$tries" -ge 20 ]; then + stop "Couldn't remove the read-only roles." "$(fix "$output" "$role")" "$output" + fi + sleep 3 + done } ` @@ -232,16 +269,16 @@ const METRIC_APIS = [ "cloudresourcemanager.googleapis.com", ] -// Signed in as a person in Cloud Shell, gcloud's calls are counted against Google's own project, -// which has the Cloud Resource Manager API. With it off for a service account, every describe +// gcloud's calls are counted against the host project (CLOUDSDK_CORE_PROJECT), or for a service +// account against its own project. With the Cloud Resource Manager API off there, every describe // and role grant fails. const apisOff = ( scopeType: GcpScopeType, apis: ReadonlyArray, ) => `# Which of the APIs this script needs are off in ${hostProject(scopeType)}, asked once. APIS_OFF="$(off ${apis.join(" ")})" -# gcloud needs the Cloud Resource Manager API of the project its calls are counted against. For a -# service account that is its own project, usually this one, so it is switched on before the +# gcloud needs the Cloud Resource Manager API of the project its calls are counted against: this +# one, or a service account's own project, usually this one too. So it is switched on before the # first read. Not everyone may switch an API on: a refusal here is reported further down. case "$APIS_OFF" in *cloudresourcemanager*) if gcloud services enable cloudresourcemanager.googleapis.com --project="$PROJECT_ID" >/dev/null 2>&1; then @@ -302,15 +339,24 @@ const PERMISSION_CHECK = ` # is not a refusal: the steps below then report for themselves. CHECKED=1 allowed() { - local resource="$1" where="$2" ask="$3" answer permission lacking="" held=0 + local resource="$1" where="$2" ask="$3" answer permission lacking="" held=0 nl=$'\\n' shift 3 - if ! answer="$(curl -fsS --max-time 15 -X POST -H 'Content-Type: application/json' \\ + if ! answer="$(curl -sS --max-time 15 -w '\\n%{http_code}' -X POST -H 'Content-Type: application/json' \\ -H @<(printf 'Authorization: Bearer %s\\n' "$TOKEN") \\ --data "{\\"permissions\\":[$(printf '"%s",' "$@" | sed 's/,$//')]}" \\ "https://cloudresourcemanager.googleapis.com/v3/$resource:testIamPermissions" 2>/dev/null)"; then CHECKED=0 return 0 fi + # Google refuses to answer for a folder or organization that this account can't see, or that + # does not exist: as good a sign of a wrong ID as having none of the permissions. + case "\${answer##*$nl}" in + 200) answer="\${answer%$nl*}" ;; + 403 | 404) + case "$answer" in *SERVICE_DISABLED* | *"has not been used"*) CHECKED=0; return 0 ;; esac + stop "$ACCOUNT can't see $where." "Check the ID in Maple. ${WRONG_ID} If it is right, this account has no rights there. $ask" ;; + *) CHECKED=0; return 0 ;; + esac for permission in "$@"; do case "$answer" in *"\\"$permission\\""*) held=1 ;; *) lacking="$lacking, $permission" ;; esac done @@ -328,11 +374,19 @@ const access = (scopeType: GcpScopeType, checks: AccessChecks): string => ` section "Checking access" command -v gcloud >/dev/null 2>&1 || stop "gcloud is not installed here." "Run this script in Cloud Shell: https://shell.cloud.google.com" +# Before 470, \`gcloud logging sinks update\` has no --include-children and every re-run fails. +GCLOUD_VERSION="$(gcloud version --format='value("Google Cloud SDK")' 2>/dev/null || true)" +case "$GCLOUD_VERSION" in [0-9].* | [0-9][0-9].* | [0-3][0-9][0-9].* | 4[0-6][0-9].*) + stop "gcloud $GCLOUD_VERSION is too old for this script." "Update it (gcloud components update, or through the package manager that installed it), or run the script in Cloud Shell: https://shell.cloud.google.com" ;; +esac ACCOUNT="$(gcloud config get-value account 2>/dev/null || true)" TOKEN="$(gcloud auth print-access-token 2>/dev/null || true)" if [ -z "$ACCOUNT" ] || [ -z "$TOKEN" ]; then stop "You are not signed in to Google Cloud." "In Cloud Shell, click Authorize when it asks. Elsewhere, run: gcloud auth login" fi +# With impersonation set, gcloud acts as that service account: it is the one that needs the roles. +IMPERSONATED="$(gcloud config get-value auth/impersonate_service_account 2>/dev/null || true)" +[ -z "$IMPERSONATED" ] || ACCOUNT="$IMPERSONATED" ok "Signed in as $ACCOUNT" ${checks.apis.length === 0 ? "" : apisOff(scopeType, checks.apis)}${scopeType === "project" ? "" : openScope(scopeType, checks.scopeMustOpen)}if ! PROJECT_NAME="$(gcloud projects describe "$PROJECT_ID" --format='value(name)' 2>/dev/null)"; then stop "Can't open project $PROJECT_ID as $ACCOUNT." "Check the ID in Maple (it is the project ID, not the name or number; list yours with: gcloud projects list). ${WRONG_ID} If it is right, this account has no access to the project." "$(gcloud projects describe "$PROJECT_ID" 2>&1 || true)" @@ -343,6 +397,7 @@ ${ ? `case "$(gcloud billing projects describe "$PROJECT_ID" --format='value(billingEnabled)' 2>/dev/null || true)" in True) ok "Billing is enabled" ;; False) stop "Project $PROJECT_ID has no billing account." "Cloud Monitoring only answers for projects with billing. Link one at https://console.cloud.google.com/billing/linkedaccount?project=$PROJECT_ID" ;; + *) note "Couldn't tell whether billing is on. Cloud Monitoring only answers for projects with billing." ;; esac ` : "" @@ -357,7 +412,11 @@ esac .join("")}# Switching an API on takes a right of its own, asked about only when one is off. [ -z "$APIS_OFF" ] || allowed "projects/$PROJECT_ID" "project $PROJECT_ID" "This script has to switch on:$APIS_OFF. Ask for Service Usage Admin there, or have an administrator switch them on." \\ serviceusage.services.enable -[ "$CHECKED" = 0 ] || ok "$ACCOUNT has the permissions this script needs" +if [ "$CHECKED" = 1 ]; then + ok "$ACCOUNT has the permissions this script needs" +else + note "Couldn't ask Google which permissions $ACCOUNT holds. A step it may not take says so." +fi ` }${checks.sink === undefined ? "" : sinkAndFilter(checks.sink)}` @@ -373,19 +432,25 @@ const permissionChecks = ( const host = [ "resourcemanager.projects.getIamPolicy", "resourcemanager.projects.setIamPolicy", + // The gets: a resource the caller may not look at is taken for missing and created again, + // which Google refuses as a duplicate. ...(logs ? [ + "pubsub.topics.get", "pubsub.topics.create", "pubsub.topics.attachSubscription", "pubsub.topics.setIamPolicy", "pubsub.topics.publish", + "pubsub.subscriptions.get", "pubsub.subscriptions.create", "pubsub.subscriptions.update", ] : []), - ...(metrics ? ["iam.serviceAccounts.create", "iam.serviceAccounts.setIamPolicy"] : []), + ...(metrics + ? ["iam.serviceAccounts.get", "iam.serviceAccounts.create", "iam.serviceAccounts.setIamPolicy"] + : []), ] - const sink = logs ? ["logging.sinks.create", "logging.sinks.update"] : [] + const sink = logs ? ["logging.sinks.get", "logging.sinks.create", "logging.sinks.update"] : [] const askHost = `Ask for Owner there, ${administrator}` if (scopeType === "project") { return [["projects/$PROJECT_ID", "project $PROJECT_ID", askHost, [...host, ...sink]]] @@ -402,6 +467,34 @@ const permissionChecks = ( ] } +// Setup only: granting to a service account Google may not have published yet, its own for the +// log sink or the reader this script creates. +const GRANT = ` +# grant +# Waited out, for a minute at most: a change that raced another one, and a service account Google +# has not published yet. Until then it answers that the account does not exist, or for the policy +# of a reader this run created (gcloud iam service-accounts ...), that the caller may not change it. +WAITED=0 +grant() { + local failed="$1" role="$2" output tries=0 + shift 2 + while ! output="$("$@" 2>&1)"; do + tries=$((tries + 1)) + conflict "$output" || case "$output" in + *"does not exist"* | *NOT_FOUND*) ;; + *PERMISSION_DENIED*) { [ "$ACCOUNT_CREATED" = 1 ] && [ "$3" = service-accounts ]; } || tries=20 ;; + *) tries=20 ;; + esac + if [ "$tries" -ge 20 ]; then stop "$failed" "$(fix "$output" "$role")" "$output"; fi + if [ "$WAITED" = 0 ] && ! conflict "$output"; then + printf ' \\342\\200\\246 waiting for Google to publish the new service account\\n' + WAITED=1 + fi + sleep 3 + done +} +` + const logsSetup = (scopeType: GcpScopeType): string => ` # ---- Log forwarding: on ---- section "Log forwarding" @@ -426,7 +519,19 @@ SUBSCRIPTION_FLAGS=( --ack-deadline=30 --message-retention-duration=1d --expiration-period=never --min-retry-delay=10s --max-retry-delay=600s ) -if exists gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then +# A subscription outlives a deleted topic, and one that was detached stays detached: neither gets +# a message again, and its topic can't be changed. Such a subscription is made anew. +subscribed() { + gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID" --format="value($1)" 2>/dev/null || true +} +SUBSCRIBED="$(subscribed topic)" +if [ -n "$SUBSCRIBED" ] && + { [ "$SUBSCRIBED" != "projects/$PROJECT_ID/topics/$TOPIC" ] || [ "$(subscribed detached)" = True ]; }; then + run "Push subscription cut off from the topic deleted" "Couldn't delete the push subscription." "$OWNER" \\ + gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" --quiet + SUBSCRIBED="" +fi +if [ -n "$SUBSCRIBED" ]; then run "Push subscription up to date" "Couldn't update the push subscription." "$OWNER" \\ gcloud pubsub subscriptions update "$SUBSCRIPTION" "\${SUBSCRIPTION_FLAGS[@]}" else @@ -434,14 +539,15 @@ else gcloud pubsub subscriptions create "$SUBSCRIPTION" --topic="$TOPIC" "\${SUBSCRIPTION_FLAGS[@]}" fi +# --no-disabled: a sink someone paused forwards again, as log forwarding is on in Maple. if [ "$SINK_EXISTS" = 1 ]; then if [ "$LOG_FILTER_MODE" = set ]; then run "Log sink up to date (filter replaced)" "Couldn't update the log sink." "$SINK_ROLE" \\ - gcloud logging sinks update "$SINK" "$DESTINATION" --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} + gcloud logging sinks update "$SINK" "$DESTINATION" --no-disabled --log-filter="$LOG_FILTER" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} FILTER_REPLACED=1 else run "Log sink up to date (filter kept)" "Couldn't update the log sink." "$SINK_ROLE" \\ - gcloud logging sinks update "$SINK" "$DESTINATION" ${SCOPES[scopeType].sink}${includeChildren(scopeType)} + gcloud logging sinks update "$SINK" "$DESTINATION" --no-disabled ${SCOPES[scopeType].sink}${includeChildren(scopeType)} fi else run "Log sink created" "Couldn't create the log sink." "$SINK_ROLE" \\ @@ -455,12 +561,16 @@ if ! WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType WRITER_IDENTITY="$(gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink} 2>&1 || true)" stop "Couldn't read the log sink back." "$(fix "$WRITER_IDENTITY" "$SINK_ROLE")" "$WRITER_IDENTITY" fi -run "Sink allowed to write to the project" "Couldn't grant the sink Logs Writer." "$OWNER" \\ +[ -n "$WRITER_IDENTITY" ] || + stop "Google named no identity for the log sink to write as." "Wait a minute and paste the script again." +grant "Couldn't grant the sink Logs Writer." "$OWNER" \\ gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ --member="$WRITER_IDENTITY" --role=roles/logging.logWriter --condition=None -run "Sink allowed to publish to the topic" "Couldn't grant the sink Pub/Sub Publisher." "$OWNER" \\ +ok "Sink allowed to write to the project" +grant "Couldn't grant the sink Pub/Sub Publisher." "$OWNER" \\ gcloud pubsub topics add-iam-policy-binding "$TOPIC" --project="$PROJECT_ID" \\ --member="$WRITER_IDENTITY" --role=roles/pubsub.publisher +ok "Sink allowed to publish to the topic" notify "Check message sent to Maple through the topic" '"logs":true' ` @@ -473,21 +583,18 @@ const logsRemoval = (scopeType: GcpScopeType, notifies: boolean): string => ` # an identity every other sink there shares. section "Log forwarding" if remains "the log sink" "$SINK_ROLE" gcloud logging sinks describe "$SINK" ${SCOPES[scopeType].sink}; then - run "Log sink deleted" "Couldn't delete the log sink." "$SINK_ROLE" \\ + remove "Log sink deleted" "Couldn't delete the log sink." "$SINK_ROLE" \\ gcloud logging sinks delete "$SINK" ${SCOPES[scopeType].sink} --quiet - REMOVED=1 printf ' \\342\\200\\246 waiting a minute for Google to stop routing to the topic\\n' sleep 60 fi if remains "the push subscription" "$OWNER" gcloud pubsub subscriptions describe "$SUBSCRIPTION" --project="$PROJECT_ID"; then - run "Push subscription deleted" "Couldn't delete the push subscription." "$OWNER" \\ + remove "Push subscription deleted" "Couldn't delete the push subscription." "$OWNER" \\ gcloud pubsub subscriptions delete "$SUBSCRIPTION" --project="$PROJECT_ID" --quiet - REMOVED=1 fi if remains "the topic" "$OWNER" gcloud pubsub topics describe "$TOPIC" --project="$PROJECT_ID"; then - run "Topic deleted" "Couldn't delete the Pub/Sub topic." "$OWNER" \\ + remove "Topic deleted" "Couldn't delete the Pub/Sub topic." "$OWNER" \\ gcloud pubsub topics delete "$TOPIC" --project="$PROJECT_ID" --quiet - REMOVED=1 fi [ "$REMOVED" = 1 ] || ok "Nothing left to remove" ${notifies ? `notify "Maple notified" '"logs":false'\n` : ""}` @@ -519,25 +626,6 @@ else ACCOUNT_CREATED=1 fi -# grant -# Google can take a minute to publish a new service account, and until then answers that it does -# not exist. Only that answer is waited out, for a minute at most. -WAITED=0 -grant() { - local failed="$1" role="$2" output tries=0 - shift 2 - while ! output="$("$@" 2>&1)"; do - tries=$((tries + 1)) - case "$output" in *"does not exist"* | *NOT_FOUND*) ;; *) tries=20 ;; esac - if [ "$tries" -ge 20 ]; then stop "$failed" "$(fix "$output" "$role")" "$output"; fi - if [ "$WAITED" = 0 ]; then - printf ' \\342\\200\\246 waiting for Google to publish the new service account\\n' - WAITED=1 - fi - sleep 3 - done -} - # Read-only roles on the ${scopeType}${scopeType === "project" ? "" : ", inherited by every project under it"}: # roles/monitoring.viewer read metrics (monitoring.timeSeries.list) # roles/cloudasset.viewer list resources (cloudasset.assets.searchAllResources) @@ -548,7 +636,8 @@ for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do ${SCOPES[scopeType].gcloud} add-iam-policy-binding "$SCOPE_ID" \\ ${serviceAccountMember} --role="$ROLE" --condition=None done -# Cloud Asset Inventory checks serviceusage.services.use on the project its API is called through. +# Maple's reads name this project as the one their quota is counted against, which takes +# serviceusage.services.use on it. grant "Couldn't grant the read-only roles." "$OWNER" \\ gcloud projects add-iam-policy-binding "$PROJECT_ID" \\ ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None @@ -576,7 +665,7 @@ account_remains() { return fi output="$("$@" 2>&1 || true)" - case "$output" in *"has not been used"* | *"is disabled"*) return 1 ;; esac + case "$output" in *"has not been used"* | *"is disabled"*) api_off "the read-only service account"; return 1 ;; esac stop "Couldn't check whether the read-only service account still exists." "$(fix "$output" "$OWNER")" "$output" } if account_remains gcloud iam service-accounts list --project="$PROJECT_ID" \\ @@ -584,14 +673,13 @@ if account_remains gcloud iam service-accounts list --project="$PROJECT_ID" \\ # Role bindings first: once the account is deleted they can no longer be removed by name. for ROLE in roles/monitoring.viewer roles/cloudasset.viewer; do unbind "$IAM_ROLE" ${SCOPES[scopeType].gcloud} remove-iam-policy-binding "$SCOPE_ID" \\ - ${serviceAccountMember} --role="$ROLE" --condition=None + ${serviceAccountMember} --role="$ROLE" --all done unbind "$OWNER" gcloud projects remove-iam-policy-binding "$PROJECT_ID" \\ - ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --condition=None + ${serviceAccountMember} --role=roles/serviceusage.serviceUsageConsumer --all ok "Read-only roles removed" - run "Read-only service account deleted" "Couldn't delete the service account." "$OWNER" \\ + remove "Read-only service account deleted" "Couldn't delete the service account." "$OWNER" \\ gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --project="$PROJECT_ID" --quiet - REMOVED=1 else ok "Nothing left to remove" fi @@ -600,19 +688,25 @@ ${notifies ? `notify "Maple notified" '"metrics":false'\n` : ""}` /** * The text that is copied: the script as a here-document for a child bash. Pasted at a prompt, * nothing runs before the closing brace is read, a failing step ends the child and not the login - * shell, and the terminal stays the script's stdin. The first line drops the paste from the - * history of bash 5; its leading space does the same under `HISTCONTROL=ignorespace`. Nothing + * shell, and the terminal stays the script's stdin. The first line keeps the paste, secret and + * all, out of the history file of zsh 5.3+ (which writes it after the command ran) and drops it + * from the history of bash 5; its leading space does the same under `HISTCONTROL=ignorespace` + * and `HIST_IGNORE_SPACE`. bash 3.2 keeps only the first line, which has no secret. Nothing * outside the here-document may be a comment: an interactive zsh runs `#` as a command. */ +// A terminal, not the bytes, decides where a pasted line ends: a CR or a TAB (completion) in a +// value could end the here-document early and hand the rest to the login shell. +const PASTE_SAFE = /^[\x20-\x7e\n]*$/u + const forPaste = (marker: string, body: string): Effect.Effect => - body.split("\n").includes(marker) + !PASTE_SAFE.test(body) || body.split("\n").includes(marker) ? Effect.fail( new IntegrationsPersistenceError({ - message: "A Google Cloud script value would end the script's here-document", + message: "A Google Cloud script value would end the script's here-document early", }), ) : Effect.succeed( - ` { [ -z "\${BASH_VERSION-}" ] || case "$(history 1)" in *${marker}*) history -d -1 ;; esac 2>/dev/null + ` { [ -z "\${ZSH_VERSION-}" ] || case "\${HISTORY_IGNORE-}" in *"*${marker}*"*) ;; *) HISTORY_IGNORE="(\${HISTORY_IGNORE:+$HISTORY_IGNORE|}*${marker}*)" ;; esac; [ -z "\${BASH_VERSION-}" ] || case "$(history 1)" in *${marker}*) history -d -1 ;; esac 2>/dev/null bash /dev/fd/3 3<<'${marker}' ${body}${marker} } @@ -657,7 +751,8 @@ export const renderGcpSetupScript = (input: GcpSetupScriptInput) => { # Safe to run again: it sets up what is on and removes what is off. # Keep it private: PUSH_ENDPOINT contains this connection's secret. set -euo pipefail -export CLOUDSDK_CORE_DISABLE_PROMPTS=1 +# gcloud would log every argument, the secret too, and the access token to ~/.config/gcloud/logs. +export CLOUDSDK_CORE_DISABLE_PROMPTS=1 CLOUDSDK_CORE_DISABLE_FILE_LOGGING=1 ${ logs ? ` @@ -689,7 +784,10 @@ LOG_FILTER=${sh(gcpLogFilter(input.logFilter === "include_gke_container_logs"))} FILTER_REPLACED=0 ACCOUNT_CREATED=0 REMOVED=0 -${output("Setup did not finish. Nothing needs undoing: fix this and paste the script again.")}${notify(logs)} +${output( + "Setup did not finish. Nothing needs undoing: fix this and paste the script again.", + "Setup interrupted. Nothing needs undoing: paste the script again to finish it.", +)}${notify(logs)} # exists : a look that fails for any reason falls through to the create, # which reports for itself. exists() { "$@" >/dev/null 2>&1; } @@ -719,7 +817,7 @@ apis() { fi ok "APIs enabled ($names)" } -${logs && metrics === "on" ? "" : REMOVAL_HELPERS} +${logs || metrics === "on" ? GRANT : ""}${logs && metrics === "on" ? "" : REMOVAL_HELPERS} printf 'Maple setup for %s\\n' "$SCOPE" printf ' Log forwarding %s\\n Metrics and resources %s\\n' ${sh(state(logs))} ${sh(metricsState)} ${access(scopeType, { @@ -768,10 +866,13 @@ export const renderGcpCleanupScript = (input: GcpCleanupScriptInput) => # Removes what the Maple setup script created. The APIs it switched on stay on. # Safe to run again.${input.pushEndpoint === undefined ? "" : "\n# Keep it private: PUSH_ENDPOINT contains this connection's secret."} set -euo pipefail -export CLOUDSDK_CORE_DISABLE_PROMPTS=1 +# gcloud would log every argument, the secret too, and the access token to ~/.config/gcloud/logs. +export CLOUDSDK_CORE_DISABLE_PROMPTS=1 CLOUDSDK_CORE_DISABLE_FILE_LOGGING=1 ${variables(input, input.pushEndpoint === undefined ? [] : [`PUSH_ENDPOINT=${sh(input.pushEndpoint)}`])}REMOVED=0 +ANY_API_OFF_MATTERS=1 ${output( "Cleanup did not finish. Fix this and paste the script again: it continues where it stopped.", + "Cleanup interrupted. Paste the script again: it continues where it stopped.", )}${REMOVAL_HELPERS} printf 'Maple cleanup for %s\\n' "$SCOPE" ${access(input.scopeType, { apis: [], sink: undefined, scopeMustOpen: false, billing: false, permissions: [] })}${logsRemoval(input.scopeType, false)}${metricsRemoval(input.scopeType, false)}${ @@ -782,7 +883,11 @@ ${access(input.scopeType, { apis: [], sink: undefined, scopeMustOpen: false, bil ${POST_REPORT} --data "$(report '"logs":false,"metrics":false')" "$PUSH_ENDPOINT" >/dev/null 2>&1 || true ` } -printf '\\nDone. Everything the setup script created is gone.\\n' +if [ "$UNCHECKED" = 1 ]; then + printf '\\nDone, but not all of it could be checked: see above.\\n' +else + printf '\\nDone. Everything the setup script created is gone.\\n' +fi printf ' Left in place: the APIs it switched on, and the Logs Writer role of Google'"'"'s logging\\n' printf ' service account on %s, which other sinks share.\\n' "$PROJECT_ID" ${input.pushEndpoint === undefined ? "" : `printf ' Now disconnect in Maple: %s\\n' "$MAPLE_URL"\n`}`, From 56947728715e8a5a2b50a5c3043b3f6c4ef86a43 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 15:22:35 +0200 Subject: [PATCH 21/21] chore(db): regenerate the gcp_connectors migration after main's newer migrations --- .../migration.sql | 0 .../snapshot.json | 42 ++++++++++++++----- 2 files changed, 31 insertions(+), 11 deletions(-) rename packages/db/drizzle/{20261014195136_gcp_connectors => 20261015132217_gcp_connectors}/migration.sql (100%) rename packages/db/drizzle/{20261014195136_gcp_connectors => 20261015132217_gcp_connectors}/snapshot.json (99%) diff --git a/packages/db/drizzle/20261014195136_gcp_connectors/migration.sql b/packages/db/drizzle/20261015132217_gcp_connectors/migration.sql similarity index 100% rename from packages/db/drizzle/20261014195136_gcp_connectors/migration.sql rename to packages/db/drizzle/20261015132217_gcp_connectors/migration.sql diff --git a/packages/db/drizzle/20261014195136_gcp_connectors/snapshot.json b/packages/db/drizzle/20261015132217_gcp_connectors/snapshot.json similarity index 99% rename from packages/db/drizzle/20261014195136_gcp_connectors/snapshot.json rename to packages/db/drizzle/20261015132217_gcp_connectors/snapshot.json index 5dc36ffd0f..c1ae4c8ed1 100644 --- a/packages/db/drizzle/20261014195136_gcp_connectors/snapshot.json +++ b/packages/db/drizzle/20261015132217_gcp_connectors/snapshot.json @@ -1,9 +1,9 @@ { "version": "1", "dialect": "postgres", - "id": "5c9b427359d9ed071f9315c14436e3a7a45982daa38c8e68fd82bca545017584", + "id": "07cb1858dcef99be1aaa58d1cd8109f0ed5e6d6f979a00467ef0a2c42926b57b", "prevIds": [ - "bc5f085e30340d63b371a71ec52b98b88451aec4818d283f307d251318828cbf" + "825059598ee5dc6c0dcb9a96c757752ad3f35c4bd83401351d640f65239930b6" ], "entities": [ { @@ -3448,8 +3448,18 @@ { "kind": "column", "table": "cloudflare_analytics_state", - "name": "last_success_at", + "name": "grant_call_cap", "position": 14, + "type": "integer", + "notNull": false, + "default": null, + "identity": null + }, + { + "kind": "column", + "table": "cloudflare_analytics_state", + "name": "last_success_at", + "position": 15, "type": "timestamp with time zone", "notNull": false, "default": null, @@ -3459,7 +3469,7 @@ "kind": "column", "table": "cloudflare_analytics_state", "name": "last_error", - "position": 15, + "position": 16, "type": "text", "notNull": false, "default": null, @@ -3469,7 +3479,7 @@ "kind": "column", "table": "cloudflare_analytics_state", "name": "last_error_at", - "position": 16, + "position": 17, "type": "timestamp with time zone", "notNull": false, "default": null, @@ -3479,7 +3489,7 @@ "kind": "column", "table": "cloudflare_analytics_state", "name": "lease_until", - "position": 17, + "position": 18, "type": "timestamp with time zone", "notNull": false, "default": null, @@ -3489,7 +3499,7 @@ "kind": "column", "table": "cloudflare_analytics_state", "name": "created_at", - "position": 18, + "position": 19, "type": "timestamp with time zone", "notNull": true, "default": null, @@ -3499,7 +3509,7 @@ "kind": "column", "table": "cloudflare_analytics_state", "name": "updated_at", - "position": 19, + "position": 20, "type": "timestamp with time zone", "notNull": true, "default": null, @@ -10758,8 +10768,18 @@ { "kind": "column", "table": "vcs_installations", - "name": "installed_by_user_id", + "name": "ci_traces_enabled", "position": 11, + "type": "boolean", + "notNull": true, + "default": "true", + "identity": null + }, + { + "kind": "column", + "table": "vcs_installations", + "name": "installed_by_user_id", + "position": 12, "type": "text", "notNull": true, "default": null, @@ -10769,7 +10789,7 @@ "kind": "column", "table": "vcs_installations", "name": "created_at", - "position": 12, + "position": 13, "type": "timestamp with time zone", "notNull": true, "default": null, @@ -10779,7 +10799,7 @@ "kind": "column", "table": "vcs_installations", "name": "updated_at", - "position": 13, + "position": 14, "type": "timestamp with time zone", "notNull": true, "default": null,