From 4d61ed6de9a75ad5204b970220143a522c244ba2 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Thu, 15 Oct 2026 01:38:29 +0200 Subject: [PATCH 1/8] feat(dashboards): chart Google Cloud metrics live with PromQL A widget data source kind `promql` stores a PromQL query, optional projects and a legend template. The browser and the share API both run it through GcpPromQlService as a range query with the chart's bucket as step, and draw the answer as wide chart rows. The editor gets a Google Cloud source tab with a metric search that writes a starter query. --- .../docs/dashboards/build-dashboards.md | 5 +- .../src/content/docs/integrations/gcp.md | 22 ++ apps/web/src/api/warehouse/gcp-promql.ts | 70 +++++++ .../config/promql-editor-panel.tsx | 195 ++++++++++++++++++ .../config/settings-fields.tsx | 14 +- .../config/widget-query-builder-page.tsx | 117 +++++++++-- .../config/widget-settings-bar.tsx | 8 +- .../data-source-registry.test.ts | 8 + .../dashboard-builder/data-source-registry.ts | 2 + .../list/dashboard-summary.ts | 19 +- .../toolbar/share-support.test.ts | 1 + .../toolbar/share-support.ts | 11 +- .../src/components/dashboard-builder/types.ts | 4 +- .../widgets/widget-shell.tsx | 13 +- apps/web/src/hooks/use-share-dashboard.ts | 3 +- apps/web/src/hooks/use-widget-data.ts | 35 +++- apps/web/src/lib/query-builder/panel-types.ts | 3 + .../services/atoms/warehouse-query-atoms.ts | 6 + .../DashboardWidgetDataService.test.ts | 103 ++++++++- .../dashboards/DashboardWidgetDataService.ts | 46 ++++- .../dashboards/widget-data-parity.test.ts | 10 +- .../http/v2/dashboard-widget-parity.test.ts | 11 + packages/domain/src/http/v2/dashboards.ts | 10 + packages/query-engine/src/gcp-promql.test.ts | 153 ++++++++++++++ packages/query-engine/src/gcp-promql.ts | 88 ++++++++ packages/query-engine/src/index.ts | 1 + .../query-engine/src/widget-request.test.ts | 15 ++ packages/query-engine/src/widget-request.ts | 9 +- packages/widgets/src/dashboard/access.ts | 18 ++ packages/widgets/src/dashboard/construct.ts | 16 +- packages/widgets/src/dashboard/index.ts | 11 +- .../widgets/src/dashboard/legacy-endpoints.ts | 3 + .../src/dashboard/promql-source.test.ts | 63 ++++++ packages/widgets/src/dashboard/redact.ts | 15 +- packages/widgets/src/dashboard/request.ts | 5 + .../widgets/src/dashboard/v3/data-source.ts | 19 +- 36 files changed, 1085 insertions(+), 47 deletions(-) create mode 100644 apps/web/src/api/warehouse/gcp-promql.ts create mode 100644 apps/web/src/components/dashboard-builder/config/promql-editor-panel.tsx create mode 100644 packages/query-engine/src/gcp-promql.test.ts create mode 100644 packages/query-engine/src/gcp-promql.ts create mode 100644 packages/widgets/src/dashboard/promql-source.test.ts diff --git a/apps/landing/src/content/docs/dashboards/build-dashboards.md b/apps/landing/src/content/docs/dashboards/build-dashboards.md index 973292338c..62de85b8e3 100644 --- a/apps/landing/src/content/docs/dashboards/build-dashboards.md +++ b/apps/landing/src/content/docs/dashboards/build-dashboards.md @@ -65,10 +65,11 @@ Most types come with presets you can start from. Pick one, then edit its query. ### Queries -A widget's editor has two tabs: +A widget's editor has these source tabs: - **Query Builder** reads **Traces**, **Logs**, **Metrics** or **Product events**. Pick an aggregation, add filters and a group-by, and combine queries with a formula. - **Raw SQL** runs your own SQL. The query must reference `$__orgFilter`. See the [SQL reference](/docs/reference/sql). +- **Google Cloud** charts a PromQL query over your Google Cloud Monitoring metrics, read live from Google. It is there for line, area, bar, stat and gauge widgets and needs a [Google Cloud connection](/docs/integrations/gcp). See [Live queries](/docs/integrations/gcp#live-queries). The preview updates as you edit. Save the widget to put it on the dashboard. @@ -112,7 +113,7 @@ Open **⋮ → Share…** and pick who can open the dashboard's link: | Anyone in this organization | Signed-in members of your organization, from the link. | | Anyone with the link | Anyone who has the link, without signing in. They see the dashboard and its data. | -**Replace** issues a new link and stops the old one working. Shared views render query builder charts, raw SQL, notes, funnels, paths, and the errors, service and logs widgets. The dialog names any widget that will not render for viewers. +**Replace** issues a new link and stops the old one working. Shared views render query builder charts, raw SQL, Google Cloud PromQL charts, notes, funnels, paths, and the errors, service and logs widgets. The dialog names any widget that will not render for viewers. To put a single chart into another page, see [Embed dashboard charts](/docs/dashboards/embed-charts). diff --git a/apps/landing/src/content/docs/integrations/gcp.md b/apps/landing/src/content/docs/integrations/gcp.md index 801688a8a4..1ea3234059 100644 --- a/apps/landing/src/content/docs/integrations/gcp.md +++ b/apps/landing/src/content/docs/integrations/gcp.md @@ -417,6 +417,28 @@ Until a connection's setup script has run, the page reads **Finish setting up Go With **Metrics and resources** off on every connection, the page leaves the sidebar and the overview. Its address still opens it: for a time range with data it shows the service tabs with what Maple collected before, under the notice **Google Cloud metrics are switched off**, and otherwise it reads **Turn on metrics for Google Cloud**. The **Resources** tab is back once a connection collects again. +## Live queries + +A dashboard widget can chart any Cloud Monitoring metric with a PromQL query, read from Google each time the widget loads. Nothing is stored in Maple. Live queries work on every connection, whether or not it collects **Metrics and resources**. + +To add one, edit a widget, choose a line, area, bar, stat or gauge panel, and pick **Google Cloud** as the source. **Start from a metric** searches the metric names of your projects and writes a starter query for the one you pick: a rate for counters, an average for gauges, the 95th percentile for distributions. Click **Run preview** to read it. + +Write metric names in PromQL's UTF-8 form, in quotes inside the braces: + +```promql +sum by (service_name) (rate({"run.googleapis.com/request_count", monitored_resource="cloud_run_revision"}[5m])) +``` + +- A metric written by more than one resource type needs a `monitored_resource` matcher, as above. Without it Google refuses the query and the widget shows Google's message. +- A distribution metric is three series: `_bucket`, `_count` and `_sum`. For a percentile, use `histogram_quantile(0.95, sum by (le) (rate({"run.googleapis.com/request_latencies_bucket"}[5m])))`. +- **Legend** names each series from its labels, for example `{{service_name}}`. Without one, a series is named after all its labels. + +Maple runs the query once per project and every series carries a `project_id` label, so an aggregation such as `sum` adds up within a project, never across projects. A query reads up to 20 projects: with more, list the projects to read in **Projects**. When some projects could not be read, the widget shows the rest and names the ones missing under the chart. + +Data arrives in Cloud Monitoring within about 2 minutes. How far back a query reaches is Google's retention: 24 months for Compute Engine, GKE, Cloud SQL and a few other services, six weeks for most other Google Cloud metrics. See **Data retention** in [Cloud Monitoring quotas and limits](https://cloud.google.com/monitoring/quotas). A widget shared through a link runs its saved query; viewers can change the time range and dashboard variables, not the query. + +Google bills the time series each read returns to your Google Cloud project, separately from your Maple plan. Maple caches each answer for a minute, and older time ranges for 10 minutes. Aggregating with `sum by` or `avg by` returns fewer series and costs less. See [Google Cloud Observability pricing](https://cloud.google.com/stackdriver/pricing). + ## Google Cloud costs Google bills Pub/Sub and Cloud Monitoring usage to your account, separately from your Maple plan. diff --git a/apps/web/src/api/warehouse/gcp-promql.ts b/apps/web/src/api/warehouse/gcp-promql.ts new file mode 100644 index 0000000000..8c9e3f994f --- /dev/null +++ b/apps/web/src/api/warehouse/gcp-promql.ts @@ -0,0 +1,70 @@ +import { Effect, Schema } from "effect" +import { GcpPromQlRangeRequest } from "@maple/domain/http" +import { promQlChartRows, promQlFailureNotice, promQlStepSeconds } from "@maple/query-engine" +import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" +import { WarehouseDateTimeString, decodeInput, runWarehouseQuery } from "@/api/warehouse/effect-utils" + +// Live Google Cloud Monitoring reads through the API's PromQL proxy (widget data source +// `gcp_promql`). Nothing comes from the warehouse; the API edge-caches each answer. + +const PromQlChartInput = Schema.Struct({ + query: Schema.String, + projects: Schema.optional(Schema.Array(Schema.String)), + legend: Schema.optional(Schema.String), + startTime: WarehouseDateTimeString, + endTime: WarehouseDateTimeString, +}) + +/** A PromQL range query as chart rows, plus a one-line notice when some projects could not be read. */ +export const getGcpPromQlChart = Effect.fn("Integrations.getGcpPromQlChart")(function* ({ + data, +}: { + data: (typeof PromQlChartInput)["Encoded"] +}) { + const input = yield* decodeInput(PromQlChartInput, data, "getGcpPromQlChart") + const payload = yield* decodeInput( + GcpPromQlRangeRequest, + { + query: input.query.trim(), + startTime: input.startTime, + endTime: input.endTime, + stepSeconds: promQlStepSeconds(input.startTime, input.endTime), + ...(input.projects === undefined ? undefined : { projects: input.projects }), + }, + "getGcpPromQlChart", + ) + const result = yield* runWarehouseQuery("gcpPromQlQueryRange", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.integrations.gcpPromQlQueryRange({ payload }) + }), + ) + const notice = promQlFailureNotice(result.failures, true) + return { + data: promQlChartRows(result.series, result.projects.length, input.legend), + ...(notice === undefined ? undefined : { notice }), + } +}) + +/** Metric names containing `search`, for the PromQL editor's metric picker. */ +export const getGcpMetricSearch = Effect.fn("Integrations.getGcpMetricSearch")(function* ({ + data, +}: { + data: { readonly search: string } +}) { + return yield* runWarehouseQuery("gcpMetricSearch", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.integrations.gcpMetricSearch({ query: { search: data.search, limit: 50 } }) + }), + ) +}) + +/** What a starter query needs to know about a metric. */ +export const getGcpMetricDescriptor = (type: string) => + runWarehouseQuery("gcpMetricDescribe", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.integrations.gcpMetricDescribe({ query: { type } }) + }), + ) diff --git a/apps/web/src/components/dashboard-builder/config/promql-editor-panel.tsx b/apps/web/src/components/dashboard-builder/config/promql-editor-panel.tsx new file mode 100644 index 0000000000..5cc168d99a --- /dev/null +++ b/apps/web/src/components/dashboard-builder/config/promql-editor-panel.tsx @@ -0,0 +1,195 @@ +import { useDeferredValue, useState } from "react" +import { Effect, Exit } from "effect" +import { promQlStarterQuery } from "@maple/query-engine" +import { Badge } from "@maple/ui/components/ui/badge" +import { Button } from "@maple/ui/components/ui/button" +import { + Combobox, + ComboboxContent, + ComboboxInput, + ComboboxItem, + ComboboxList, +} from "@maple/ui/components/ui/combobox" +import { EmptyMessage } from "@maple/ui/components/ui/empty" +import { Field, FieldLabel } from "@maple/ui/components/ui/field" +import { Input } from "@maple/ui/components/ui/input" +import { TONE_TEXT } from "@maple/ui/lib/tone" +import { cn } from "@maple/ui/lib/utils" +import { getGcpMetricDescriptor } from "@/api/warehouse/gcp-promql" +import { useAsyncAction } from "@/hooks/use-mutation-action" +import { displayError } from "@/lib/error-messages" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { disabledResultAtom } from "@/lib/services/atoms/disabled-result-atom" +import { gcpMetricSearchResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" + +/** The editor's form. `projects` stays the typed text until it is saved. */ +export interface PromQlDraft { + query: string + projects: string + legend: string +} + +/** Project ids from the comma- or space-separated field; none means every project. */ +export const parsePromQlProjects = (text: string): ReadonlyArray => + text.split(/[\s,]+/u).filter((projectId) => projectId !== "") + +interface PromQlEditorPanelProps { + draft: PromQlDraft + onDraftChange: (next: PromQlDraft) => void + onRunPreview: () => void + /** The preview's failure, e.g. Google's message for a query it refused. */ + error?: { title: string; message?: string } +} + +export function PromQlEditorPanel({ draft, onDraftChange, onRunPreview, error }: PromQlEditorPanelProps) { + const [search, setSearch] = useState("") + const deferredSearch = useDeferredValue(search.trim()) + const searchResult = useAtomValue( + deferredSearch === "" + ? disabledResultAtom() + : gcpMetricSearchResultAtom({ data: { search: deferredSearch } }), + ) + const metrics = Result.builder(searchResult) + .onSuccess((response) => (response as { metrics: ReadonlyArray }).metrics) + .orElse((): ReadonlyArray => []) + + const [starterError, setStarterError] = useState(null) + const [insertStarter, inserting] = useAsyncAction(async (metric: string) => { + setStarterError(null) + const exit = await Effect.runPromiseExit(getGcpMetricDescriptor(metric)) + if (Exit.isFailure(exit)) { + setStarterError(displayError(exit.cause).message) + return + } + onDraftChange({ ...draft, query: promQlStarterQuery(metric, exit.value) }) + }) + + return ( +
+
+
+ + promql + + + Google Cloud Monitoring, live + +
+ {error && ( + + {error.title} + + )} +
+ +
+ + + Start from a metric: picking one replaces the query below + + + value={null} + onValueChange={(metric) => { + if (typeof metric === "string") void insertStarter(metric) + }} + > + setSearch(e.target.value)} + /> + + {metrics.length === 0 ? ( + + {deferredSearch === "" + ? "Type part of a metric name." + : `No metric name contains "${deferredSearch}".`} + + ) : ( + + {metrics.map((metric) => ( + + {metric} + + ))} + + )} + + + {starterError !== null && ( + {starterError} + )} + + +