feat(alerts): warning and critical tiers, recovery values and N-of-M on one rule

.github/workflows/deploy-pr-preview.yml

@@ -124,6 +124,31 @@ jobs:
                   WEB_URL: ${{ steps.deploy.outputs.web_url }}
               run: bun scripts/preview-seed-smoke.ts
 
+            # Browser e2e against the fresh preview (apps/web/e2e/README.md). Non-blocking while
+            # it beds in; `agent.*` tests register only once ANTHROPIC_API_KEY is in Infisical.
+            - name: End-to-end tests against the preview
+              id: e2e
+              if: ${{ env.TEARDOWN != 'true' && steps.smoke.outcome == 'success' }}
+              continue-on-error: true
+              working-directory: apps/web/e2e
+              env:
+                  MAPLE_E2E_APP_URL: ${{ steps.deploy.outputs.web_url }}
+                  MAPLE_E2E_API_URL: ${{ steps.deploy.outputs.api_url }}
+                  E2E_TELEMETRY_DISABLED: "1"
+              run: |
+                  bunx e2e-web install chromium --with-deps
+                  bunx e2e run --reporter list,junit
+
+            - name: Upload e2e results
+              if: ${{ !cancelled() && steps.e2e.outcome != 'skipped' }}
+              uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+              with:
+                  name: e2e-results
+                  path: apps/web/e2e/.e2e/
+                  if-no-files-found: warn
+                  include-hidden-files: true
+                  retention-days: 7
+
             - name: Destroy PR preview stack with Alchemy
               id: destroy
               if: ${{ env.TEARDOWN == 'true' }}

apps/ai/src/mcp/lib/alert-rules.ts

@@ -107,14 +107,26 @@ export const parseRuleId = (ruleId: string) =>
 		onSome: (id) => Effect.succeed(id),
 	})
 
+/**
+ * An invalid rule as the model reads it: the message and each detail once. A
+ * message that already folds in every detail is dropped as a repeat; any other
+ * message names the failure the details only qualify, so it stays.
+ */
+export const invalidRuleInput = (error: AlertValidationError) => {
+	const repeatsDetails =
+		error.details.length > 0 && error.details.every((detail) => error.message.includes(detail))
+	return new McpInvalidInputError({
+		message: [
+			repeatsDetails ? "Invalid alert rule:" : `Invalid alert rule: ${error.message}`,
+			...error.details.map((detail) => `- ${detail}`),
+		].join("\n"),
+	})
+}
+
 /** The write failures that are the caller's to fix, as input errors naming the parameter. */
 export const ruleWriteInputErrors = {
 	"@maple/http/errors/AlertValidationError": (error: AlertValidationError) =>
-		Effect.fail(
-			new McpInvalidInputError({
-				message: [`Invalid alert rule: ${error.message}`, ...error.details].join("\n"),
-			}),
-		),
+		Effect.fail(invalidRuleInput(error)),
 	"@maple/http/errors/AlertRuleDestinationNotFoundError": (error: AlertRuleDestinationNotFoundError) =>
 		Effect.fail(
 			new McpInvalidInputError({

apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts

@@ -14,6 +14,7 @@ import {
 	AlertRulePreviewResponse,
 	AlertRulesListResponse,
 	AlertRuleUpsertRequest,
+	AlertValidationError,
 	OrgId,
 	UserId,
 } from "@maple/domain/http"
@@ -33,6 +34,7 @@ import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertRead
 import { CurrentMcpTenant } from "../../lib/query-warehouse"
 import { executeRegisteredMcpToolUnscoped } from "../registry"
 import { buildUpdatedRequest } from "../update-alert-rule"
+import { invalidRuleInput } from "../../lib/alert-rules"
 import type { McpToolResult } from "../types"
 
 const RULE_ID = "7f1c2d3e-4b5a-4c6d-8e9f-0a1b2c3d4e5f"
@@ -59,6 +61,10 @@ const rule = Schema.decodeUnknownSync(AlertRuleDocument)({
 	windowMinutes: 5,
 	minimumSampleCount: 0,
 	consecutiveBreachesRequired: 1,
+	breachesOutOf: null,
+	thresholds: [
+		{ severity: "critical", value: 0.05, upper: null, recoveryValue: null, destinationIds: [DEST_ID] },
+	],
 	consecutiveHealthyRequired: 1,
 	renotifyIntervalMinutes: 60,
 	apdexThresholdMs: null,
@@ -111,6 +117,7 @@ const incident = (overrides: Record<string, unknown>) => ({
 	comparator: "gt",
 	threshold: 0.05,
 	thresholdUpper: null,
+	peakSeverity: null,
 	firstTriggeredAt: NOW,
 	lastTriggeredAt: NOW,
 	resolvedAt: null,
@@ -181,6 +188,15 @@ const layer = (seen: Seen) =>
 						windowMinutes: 5,
 						threshold: 0.05,
 						thresholdUpper: null,
+						thresholds: [
+							{
+								severity: "warning",
+								value: 0.05,
+								upper: null,
+								recoveryValue: null,
+								destinationIds: [],
+							},
+						],
 						comparator: "gt",
 						truncatedToStart: null,
 						series: [
@@ -208,7 +224,14 @@ const layer = (seen: Seen) =>
 				return Effect.succeed(
 					id === RULE_ID
 						? rule
-						: { ...rule, id, name: "Latency", destinationIds: request.destinationIds },
+						: {
+								...rule,
+								id,
+								name: "Latency",
+								destinationIds: (request.thresholds ?? []).flatMap(
+									(tier) => tier.destinationIds,
+								),
+							},
 				)
 			},
 			listDestinations: () =>
@@ -270,6 +293,7 @@ const layer = (seen: Seen) =>
 								consecutiveHealthy: 0,
 								incidentId: null,
 								incidentTransition: "none",
+								level: null,
 								evaluationDurationMs: 12,
 								errorMessage: "column Foo not found",
 								errorCategory: "validation",
@@ -293,6 +317,7 @@ const layer = (seen: Seen) =>
 								consecutiveHealthy: 0,
 								incidentId: null,
 								incidentTransition: "none",
+								level: null,
 								evaluationDurationMs: 9,
 								errorMessage: null,
 								errorCategory: null,
@@ -564,7 +589,7 @@ describe("alert tools", () => {
 		await ok("preview_alert_rule", { rule_id: RULE_ID, threshold: 0.2 }, seen)
 		expect(seen.previewed?.rule).toMatchObject({
 			name: "Checkout errors",
-			threshold: 0.2,
+			thresholds: [{ value: 0.2 }],
 			signalType: "error_rate",
 		})
 
@@ -580,7 +605,7 @@ describe("alert tools", () => {
 			seen,
 		)
 		expect(seen.updated).toMatchObject({
-			threshold: 0.1,
+			thresholds: [{ severity: "critical", value: 0.1 }],
 			name: "Checkout errors",
 			serviceNames: ["checkout"],
 			groupBy: ["service.name", "attr.http.route"],
@@ -606,9 +631,96 @@ describe("alert tools", () => {
 			...rule,
 			comparator: "between",
 			thresholdUpper: 10,
+			thresholds: [
+				{ severity: "critical", value: 0.05, upper: 10, recoveryValue: null, destinationIds: [] },
+			],
+		})
+		const upperAfter = (params: Parameters<typeof buildUpdatedRequest>[1]) =>
+			Effect.runSync(buildUpdatedRequest(range, params)).thresholds[0].upper
+		expect(upperAfter({ comparator: "gt" })).toBeNull()
+		expect(upperAfter({ threshold: 2 })).toBe(10)
+	})
+
+	it("update_alert_rule drops a recovery value the new threshold invalidates", () => {
+		const recovering = Schema.decodeUnknownSync(AlertRuleDocument)({
+			...rule,
+			comparator: "gt",
+			thresholds: [
+				{
+					severity: "critical",
+					value: 100,
+					upper: null,
+					recoveryValue: 80,
+					destinationIds: [DEST_ID],
+				},
+			],
+		})
+		const recoveryAfter = (params: Parameters<typeof buildUpdatedRequest>[1]) =>
+			Effect.runSync(buildUpdatedRequest(recovering, params)).thresholds[0].recoveryValue
+		expect(recoveryAfter({ threshold: 120 })).toBe(80)
+		expect(recoveryAfter({ threshold: 70 })).toBeNull()
+		expect(recoveryAfter({ comparator: "lt" })).toBeNull()
+	})
+
+	it("update_alert_rule keeps a two-tier rule's tiers and refuses to merge them", () => {
+		const tiered = Schema.decodeUnknownSync(AlertRuleDocument)({
+			...rule,
+			breachesOutOf: 5,
+			thresholds: [
+				{
+					severity: "warning",
+					value: 0.02,
+					upper: null,
+					recoveryValue: 0.01,
+					destinationIds: ["00000000-0000-4000-8000-00000000d001"],
+				},
+				{
+					severity: "critical",
+					value: 0.05,
+					upper: null,
+					recoveryValue: null,
+					destinationIds: ["00000000-0000-4000-8000-00000000d002"],
+				},
+			],
 		})
-		expect(buildUpdatedRequest(range, { comparator: "gt" }).thresholdUpper).toBeNull()
-		expect(buildUpdatedRequest(range, { threshold: 2 }).thresholdUpper).toBe(10)
+		const renamed = Effect.runSync(buildUpdatedRequest(tiered, { name: "Renamed" }))
+		expect(renamed.thresholds).toEqual(tiered.thresholds)
+		expect(renamed.breachesOutOf).toBe(5)
+
+		const added = Effect.runSync(
+			buildUpdatedRequest(tiered, {
+				add_destination_ids: ["00000000-0000-4000-8000-00000000d003"],
+				remove_destination_ids: ["00000000-0000-4000-8000-00000000d002"],
+			}),
+		)
+		expect(added.thresholds.map((tier) => tier.destinationIds)).toEqual([
+			["00000000-0000-4000-8000-00000000d001", "00000000-0000-4000-8000-00000000d003"],
+			[],
+		])
+
+		const collapsing = Effect.runSync(Effect.flip(buildUpdatedRequest(tiered, { threshold: 0.1 })))
+		expect(collapsing.parameter).toBe("threshold")
+	})
+
+	it("names each invalid-rule detail once, though the domain message already carries them", () => {
+		const details = ["threshold must be a finite number", "at least one destination must be selected"]
+		const message = invalidRuleInput(
+			new AlertValidationError({ message: `Invalid alert rule: ${details.join("; ")}`, details }),
+		).message
+		for (const detail of details) {
+			expect(message.split(detail)).toHaveLength(2)
+		}
+	})
+
+	it("keeps an invalid-rule message that names a failure its details only qualify", () => {
+		const message = invalidRuleInput(
+			new AlertValidationError({
+				message: "Unsupported traces metric: p95",
+				details: ["Traces filter http.status_code supports only =; ignoring !="],
+			}),
+		).message
+		expect(message).toContain("Unsupported traces metric: p95")
+		expect(message).toContain("ignoring !=")
 	})
 
 	it("delete_alert_rule requires confirm and maps not-found to rule_id", async () => {
@@ -693,7 +805,7 @@ describe("alert tools", () => {
 			seen,
 		)
 		expect(seen.updatedIds).toEqual([RULE_ID, ALL_ID, RAW_ID, OTHER_RAW_ID])
-		expect(seen.updated?.destinationIds).toEqual([NEW_DEST])
+		expect(seen.updated?.thresholds?.[0]?.destinationIds).toEqual([NEW_DEST])
 		const output = Schema.decodeUnknownSync(UpdateAlertRuleOutput)(result.structuredContent)
 		expect(output.rules).toHaveLength(4)
 		expect(text(result)).toContain("Updated 4 Alert Rules")

apps/ai/src/mcp/tools/preview-alert-rule.ts

@@ -9,7 +9,7 @@ import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesServ
 import { toMcpHttpError } from "../lib/map-http-error"
 import { warehouseReadToMcpHandlers } from "../lib/map-warehouse-error"
 import { CurrentMcpTenant } from "../lib/query-warehouse"
-import { formatCondition, ruleNotFound } from "../lib/alert-rules"
+import { formatCondition, invalidRuleInput, ruleNotFound } from "../lib/alert-rules"
 import { minSampleCountWarning } from "../lib/alert-rule-evaluation"
 import { truncate } from "../lib/format"
 import * as P from "../lib/params"
@@ -135,7 +135,7 @@ export function registerPreviewAlertRuleTool(server: McpToolRegistrar) {
 					.pipe(Effect.mapError(toMcpHttpError("preview_alert_rule")))
 				const current = list.rules.find((r) => r.id === params.rule_id)
 				if (!current) return yield* ruleNotFound(params.rule_id)
-				rule = buildUpdatedRequest(current, { ...params, rule_id: current.id })
+				rule = yield* buildUpdatedRequest(current, { ...params, rule_id: current.id })
 			} else {
 				rule = yield* buildAlertRuleRequest({ ...params, name: "Preview", destination_ids: [] })
 			}
@@ -154,13 +154,7 @@ export function registerPreviewAlertRuleTool(server: McpToolRegistrar) {
 			const preview = yield* alerts.previewRule(tenant.orgId, tenant.roles, request).pipe(
 				Effect.catchTags({
 					"@maple/http/errors/AlertValidationError": (error) =>
-						Effect.fail(
-							new McpInvalidInputError({
-								message: [`Invalid alert rule: ${error.message}`, ...error.details].join(
-									"\n",
-								),
-							}),
-						),
+						Effect.fail(invalidRuleInput(error)),
 					"@maple/http/errors/AlertForbiddenError": (error) =>
 						Effect.fail(toMcpHttpError("preview_alert_rule")(error)),
 					"@maple/http/errors/QueryEngineValidationError": (error) =>