@@ -92,6 +102,8 @@ export function ErrorsLab() {
onViewChange={setView}
onSortChange={setSort}
onSeverityChange={setSeverity}
+ search={search}
+ onSearchChange={(value) => setSearch(value ?? "")}
stats={}
onRetry={() => setState("ready")}
/>
@@ -181,10 +193,8 @@ function LabStatStrip({
}
}) {
const stats = [
- { value: formatNumber(summary.totalErrors), label: "errors" },
+ { value: formatNumber(summary.totalErrors), label: "error events" },
{ value: formatErrorRate(summary.errorRate), label: "of all spans" },
- { value: formatNumber(summary.affectedServicesCount), label: "services" },
- { value: formatNumber(summary.affectedTracesCount), label: "traces" },
]
return (
@@ -194,7 +204,7 @@ function LabStatStrip({
{stat.label}
))}
- in the selected window
+ in the last 24 hours
)
}
diff --git a/apps/web/src/lib/errors/hub-params.ts b/apps/web/src/lib/errors/hub-params.ts
index edb7bb87eb..47f142425d 100644
--- a/apps/web/src/lib/errors/hub-params.ts
+++ b/apps/web/src/lib/errors/hub-params.ts
@@ -5,9 +5,21 @@
* whole list there with them.
*/
-export const HUB_VIEWS = ["open", "triage", "active", "resolved", "all"] as const
+export const HUB_VIEWS = ["open", "closed", "all"] as const
export type HubView = (typeof HUB_VIEWS)[number]
+/** Tabs the list used to have. Old links still decode and land on the tab that
+ * now holds the same issues. */
+export const LEGACY_HUB_VIEWS = ["triage", "active", "resolved"] as const
+export type LegacyHubView = (typeof LEGACY_HUB_VIEWS)[number]
+
+export const hubViewOf = (view: HubView | LegacyHubView | undefined): HubView =>
+ view === undefined || view === "triage" || view === "active"
+ ? "open"
+ : view === "resolved"
+ ? "closed"
+ : view
+
/** `last_seen` leads because it is the default: newest activity first, paged
* back through older issues. `volume` is the one sort only the warehouse can
* answer, so it is the one scoped to the time range. */
diff --git a/apps/web/src/routes/errors/index.tsx b/apps/web/src/routes/errors/index.tsx
index 116cd66020..5e82cbe8fe 100644
--- a/apps/web/src/routes/errors/index.tsx
+++ b/apps/web/src/routes/errors/index.tsx
@@ -14,9 +14,10 @@ import {
ERRORS_WINDOW,
HUB_SORTS,
HUB_VIEWS,
+ LEGACY_HUB_VIEWS,
SEVERITY_FILTERS,
+ hubViewOf,
type HubSort,
- type HubView,
type SeverityFilter,
} from "@/lib/errors/hub-params"
import { CLEARED_ERROR_FILTERS, errorFilterChips, hasErrorFilters } from "@/lib/errors/error-filter-chips"
@@ -34,7 +35,9 @@ const errorsSearchSchema = Schema.Struct({
env: Schema.optional(Schema.String),
kind: Schema.optional(IssueKind),
regressed: Schema.optional(Schema.Union([Schema.Boolean, BooleanFromStringParam])),
- view: Schema.optional(Schema.Literals(HUB_VIEWS)),
+ view: Schema.optional(Schema.Literals([...HUB_VIEWS, ...LEGACY_HUB_VIEWS])),
+ // Free-text search over type, message, label and service, applied server-side.
+ q: Schema.optional(Schema.String),
sort: Schema.optional(Schema.Literals(HUB_SORTS)),
severity: Schema.optional(Schema.Literals(SEVERITY_FILTERS)),
})
@@ -73,7 +76,7 @@ function ErrorsContent() {
navigate({ search: (prev) => ({ ...prev, ...CLEARED_ERROR_FILTERS }) })
}
- const view: HubView = search.view ?? "open"
+ const view = hubViewOf(search.view)
const sort: HubSort = search.sort ?? "last_seen"
const severity: SeverityFilter = search.severity ?? "all"
@@ -93,6 +96,7 @@ function ErrorsContent() {
env={search.env}
kind={search.kind}
regressed={search.regressed}
+ search={search.q}
onClearFilters={hasErrorFilters(search) ? clearFilters : undefined}
/>
diff --git a/apps/web/src/routes/errors/issues/index.tsx b/apps/web/src/routes/errors/issues/index.tsx
index 60731e85b7..657da7976c 100644
--- a/apps/web/src/routes/errors/issues/index.tsx
+++ b/apps/web/src/routes/errors/issues/index.tsx
@@ -17,19 +17,20 @@ export const Route = createFileRoute("/errors/issues/")({
},
})
-type HubView = "open" | "triage" | "active" | "resolved" | "all"
+type HubView = "open" | "closed" | "all"
/** The old list's `workflowState` tabs map onto the hub's views. A Map rather
* than an object because the incoming value is unvalidated search-param text. */
const VIEW_FOR_STATE = new Map([
- ["triage", "triage"],
- ["regressed", "triage"],
- ["todo", "active"],
- ["in_progress", "active"],
- ["in_review", "active"],
- ["done", "resolved"],
- ["cancelled", "resolved"],
- ["wontfix", "resolved"],
+ ["triage", "open"],
+ ["regressed", "open"],
+ ["todo", "open"],
+ ["in_progress", "open"],
+ ["in_review", "open"],
+ ["verifying", "closed"],
+ ["done", "closed"],
+ ["cancelled", "closed"],
+ ["wontfix", "closed"],
["all", "all"],
])
diff --git a/docs/error-issue-lifecycle.md b/docs/error-issue-lifecycle.md
index 8e4bf6a880..45e393eb9c 100644
--- a/docs/error-issue-lifecycle.md
+++ b/docs/error-issue-lifecycle.md
@@ -215,3 +215,18 @@ loop in verification without a human ever seeing it.
| PR links and verification windows | `packages/backend/src/services/errors/IssueFixVerificationService.ts` |
| The verification tick | `packages/backend/src/services/errors/FixVerificationTickService.ts` |
| What agents are told | `apps/ai/src/mcp/resources/instructions.ts` |
+
+## Testing it locally
+
+- **Seed exceptions:** `bun run seed:demo:env`, then `MAPLE_DEV_ENV_FILE=.env.screenshots bun dev api web ingest alerting`
+ and `bun run seed:demo -- --reset --follow`. The alerting Worker skips its crons off production unless
+ `MAPLE_ALERTING_ALLOW_NONPROD=1` is set (the screenshot env sets it).
+- **Run the tick now:** `bun run errors:tick` fires the alerting minute cron once instead of waiting
+ for it. That cron runs the alert and escalation lanes too, not only the errors tick and fix
+ verification, so alert rules evaluate (and may notify) on the same fire. `--url` targets a branch stack's `.alerting.localhost`.
+- **Measure a fingerprint change before shipping it:** `bun run errors:replay -- --file ` groups
+ a sample of real error events with today's fingerprint and with the proposal in
+ `scripts/errors-replay/candidate.ts`, checks the TS mirror is in parity with production's hashes, and
+ lists the largest merges to read for false positives. The script header has the export query; samples
+ hold customer data and stay out of the repo.
+- **The list without a backend:** `/lab/errors` renders the real list view over fixtures.
diff --git a/package.json b/package.json
index 1b59452bec..d1c33c2371 100644
--- a/package.json
+++ b/package.json
@@ -11,6 +11,8 @@
"build": "turbo build",
"dev": "bun run ./scripts/dev.ts",
"dev:signin": "bun run ./scripts/dev-signin.ts",
+ "errors:tick": "bun run ./scripts/errors-tick.ts",
+ "errors:replay": "bun run ./scripts/errors-replay.ts",
"seed:demo": "bun run ./scripts/seed-demo.ts",
"seed:demo:env": "bun run ./scripts/seed-demo/make-env.ts",
"prepare": "effect-tsgo unpatch --no-typescript --oxlint && effect-tsgo patch --no-typescript --oxlint",
diff --git a/packages/backend/src/services/errors/ErrorIssueReadModelsService.test.ts b/packages/backend/src/services/errors/ErrorIssueReadModelsService.test.ts
index e3b6228460..cef81cfb48 100644
--- a/packages/backend/src/services/errors/ErrorIssueReadModelsService.test.ts
+++ b/packages/backend/src/services/errors/ErrorIssueReadModelsService.test.ts
@@ -225,6 +225,58 @@ describe("ErrorIssueReadModelsService", () => {
}).pipe(Effect.provide(makeLayer(contexts)))
})
+ it.effect("pages closed issues server-side, past newer open ones and into the archive", () => {
+ const contexts: Array = []
+ return Effect.gen(function* () {
+ const readModels = yield* ErrorIssueReadModelsService
+ const now = yield* Clock.currentTimeMillis
+ const open = asIssueId(randomUUID())
+ const verifying = asIssueId(randomUUID())
+ const wontfix = asIssueId(randomUUID())
+ const archivedDone = asIssueId(randomUUID())
+
+ // The open issue is the newest, so a recency page of one would hold only it.
+ yield* seedIssue(ORG, open, now + 5_000, { workflowState: "triage" })
+ yield* seedIssue(ORG, verifying, now, { workflowState: "verifying" })
+ yield* seedIssue(ORG, wontfix, now - 1_000, { workflowState: "wontfix" })
+ yield* seedIssue(ORG, archivedDone, now - 2_000, {
+ workflowState: "done",
+ archivedAt: now - 1_000,
+ })
+
+ const firstClosed = yield* readModels.listIssues(ORG, { stateGroup: "closed", limit: 1 })
+ expect(firstClosed.issues.map((issue) => issue.id)).toEqual([verifying])
+
+ const closed = yield* readModels.listIssues(ORG, { stateGroup: "closed", includeArchived: true })
+ expect(closed.issues.map((issue) => issue.id)).toEqual([verifying, wontfix, archivedDone])
+
+ const opened = yield* readModels.listIssues(ORG, { stateGroup: "open", includeArchived: true })
+ expect(opened.issues.map((issue) => issue.id)).toEqual([open])
+ }).pipe(Effect.provide(makeLayer(contexts)))
+ })
+
+ it.effect("searches type, message and service case-insensitively", () => {
+ const contexts: Array = []
+ return Effect.gen(function* () {
+ const readModels = yield* ErrorIssueReadModelsService
+ const now = yield* Clock.currentTimeMillis
+ const byMessage = asIssueId(randomUUID())
+ const byService = asIssueId(randomUUID())
+ const unrelated = asIssueId(randomUUID())
+
+ yield* seedIssue(ORG, byMessage, now, { exceptionMessage: "Stripe webhook signature mismatch" })
+ yield* seedIssue(ORG, byService, now - 1_000, { serviceName: "stripe-sync" })
+ yield* seedIssue(ORG, unrelated, now - 2_000)
+
+ const found = yield* readModels.listIssues(ORG, { search: "STRIPE" })
+ expect(found.issues.map((issue) => issue.id)).toEqual([byMessage, byService])
+
+ // LIKE wildcards in the term are literal, not "match everything".
+ const literal = yield* readModels.listIssues(ORG, { search: "%" })
+ assert.lengthOf(literal.issues, 0)
+ }).pipe(Effect.provide(makeLayer(contexts)))
+ })
+
it.effect("preserves issue ownership and not-found semantics", () => {
const contexts: Array = []
return Effect.gen(function* () {
diff --git a/packages/backend/src/services/errors/ErrorIssueReadModelsService.ts b/packages/backend/src/services/errors/ErrorIssueReadModelsService.ts
index eb6b62808c..fe9d651bef 100644
--- a/packages/backend/src/services/errors/ErrorIssueReadModelsService.ts
+++ b/packages/backend/src/services/errors/ErrorIssueReadModelsService.ts
@@ -56,6 +56,9 @@ const ACTIONABLE_WORKFLOW_STATES: ReadonlyArray = [
"in_progress",
"in_review",
]
+/** The list's "Closed" set: nobody has to act. `verifying` is here because the
+ * fix has merged and the verification tick, not a person, owns what happens next. */
+const CLOSED_LIST_WORKFLOW_STATES: ReadonlyArray = ["verifying", "done", "cancelled", "wontfix"]
type IssueColumns = PG.ColumnAccessor
type IssueConditions = ($: IssueColumns) => Array
@@ -106,6 +109,8 @@ export interface IssueListFilters {
/** First seen or last regressed at or after this instant. */
readonly introducedAfter?: string
readonly actionable?: boolean
+ /** `open` = the actionable states; `closed` = verifying and the closed exits. */
+ readonly stateGroup?: "open" | "closed"
}
export interface ErrorIssueReadModelsPublicApi {
@@ -237,7 +242,10 @@ const make: Effect.Effect<
const conditions: IssueConditions = ($) => [
$.orgId.eq(orgId),
opts.workflowState ? $.workflowState.eq(opts.workflowState) : undefined,
- opts.actionable ? $.workflowState.in_(...ACTIONABLE_WORKFLOW_STATES) : undefined,
+ opts.actionable || opts.stateGroup === "open"
+ ? $.workflowState.in_(...ACTIONABLE_WORKFLOW_STATES)
+ : undefined,
+ opts.stateGroup === "closed" ? $.workflowState.in_(...CLOSED_LIST_WORKFLOW_STATES) : undefined,
opts.severity === "unset"
? $.severity.isNull()
: opts.severity
diff --git a/packages/domain/src/http/v2/error-issues.ts b/packages/domain/src/http/v2/error-issues.ts
index c74050646a..927adb239e 100644
--- a/packages/domain/src/http/v2/error-issues.ts
+++ b/packages/domain/src/http/v2/error-issues.ts
@@ -131,6 +131,9 @@ export const V2ErrorIssueDetail = Schema.Struct({
})
export type V2ErrorIssueDetail = Schema.Schema.Type
+/** Longest `search` term the list accepts; clients clamp to it rather than fail. */
+export const ERROR_ISSUE_SEARCH_MAX_LENGTH = 200
+
export const V2ErrorIssueListQuery = Schema.Struct({
...ListQuery.fields,
workflow_state: Schema.optional(WorkflowState),
@@ -156,6 +159,13 @@ export const V2ErrorIssueListQuery = Schema.Struct({
// deploy introduced, rather than everything still occurring.
introduced_after: Schema.optional(Timestamp),
actionable: Schema.optional(Schema.Literal("true")),
+ // `open`: someone still has to act (same set as `actionable`). `closed`: the
+ // work is over or handed to verification (verifying, done, cancelled, wontfix).
+ state: Schema.optional(Schema.Literals(["open", "closed"])),
+ // Case-insensitive substring of the exception type, message, label or service.
+ search: Schema.optional(Schema.String.check(Schema.isMaxLength(ERROR_ISSUE_SEARCH_MAX_LENGTH))),
+ // Done issues are archived 14 days after they close; include them.
+ include_archived: Schema.optional(Schema.Literal("true")),
sort: Schema.optional(Schema.Literals(["last_seen", "severity"])),
}).annotate({
identifier: "ErrorIssueListQuery",
diff --git a/scripts/errors-replay.ts b/scripts/errors-replay.ts
new file mode 100644
index 0000000000..34f762a58d
--- /dev/null
+++ b/scripts/errors-replay.ts
@@ -0,0 +1,136 @@
+/**
+ * Replay error events through today's fingerprint and a proposed one, and show
+ * how the issue count and grouping would change. Measure a fingerprint change
+ * here before it rotates every issue in production.
+ *
+ * bun run errors:replay # the synthetic corpus
+ * bun run errors:replay -- --file sample.json # a real sample
+ *
+ * A sample is JSONL, or the `{ "rows": [...] }` object the `run_sql` MCP tool
+ * returns. Each row: serviceName, exceptionType, statusMessage,
+ * exceptionStacktrace, and optionally count (events the row stands for) and
+ * fingerprintHash (production's hash, which turns on the parity check).
+ * One way to pull a week of variants from an org:
+ *
+ * SELECT ServiceName AS serviceName, ExceptionType AS exceptionType,
+ * StatusMessage AS statusMessage, ExceptionStacktrace AS exceptionStacktrace,
+ * toString(FingerprintHash) AS fingerprintHash, count() AS count
+ * FROM error_events WHERE $__orgFilter AND $__timeFilter(Timestamp)
+ * GROUP BY serviceName, exceptionType, statusMessage, exceptionStacktrace, fingerprintHash
+ * ORDER BY count DESC LIMIT 3 BY fingerprintHash
+ *
+ * The proposal lives in `./errors-replay/candidate.ts`. Samples hold customer
+ * data: keep them out of the repo.
+ */
+import { readFileSync } from "node:fs"
+import { parseArgs } from "node:util"
+import { Schema } from "effect"
+
+import { candidateKey, currentKey, type ReplayEvent } from "./errors-replay/candidate"
+
+const { values } = parseArgs({
+ options: {
+ file: { type: "string", default: new URL("./errors-replay/corpus.jsonl", import.meta.url).pathname },
+ top: { type: "string", default: "12" },
+ },
+})
+const top = Number(values.top)
+
+const Row = Schema.Struct({
+ serviceName: Schema.String,
+ exceptionType: Schema.String,
+ statusMessage: Schema.String,
+ exceptionStacktrace: Schema.String,
+ count: Schema.optional(Schema.Union([Schema.Number, Schema.NumberFromString])),
+ fingerprintHash: Schema.optional(Schema.String),
+})
+type Row = Schema.Schema.Type
+const decodeRows = Schema.decodeUnknownSync(Schema.Array(Row))
+const decodeEnvelope = Schema.decodeUnknownSync(Schema.Struct({ rows: Schema.Array(Schema.Unknown) }))
+const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown))
+
+const text = readFileSync(values.file ?? "", "utf8").trim()
+const raw = text.startsWith("{") && text.includes('"rows"')
+ ? decodeEnvelope(decodeJson(text)).rows
+ : text.split("\n").filter((line) => line.trim() !== "").map((line) => decodeJson(line))
+const rows = decodeRows(raw)
+
+const weight = (row: Row) => row.count ?? 1
+const events = rows.reduce((sum, row) => sum + weight(row), 0)
+
+interface Keyed {
+ readonly row: Row
+ readonly current: string
+ readonly candidate: string
+}
+const keyed: ReadonlyArray = rows.map((row) => {
+ const event: ReplayEvent = row
+ return { row, current: currentKey(event), candidate: candidateKey(event) }
+})
+
+const distinct = (keys: Iterable) => new Set(keys).size
+const currentGroups = distinct(keyed.map((k) => k.current))
+const candidateGroups = distinct(keyed.map((k) => k.candidate))
+const pct = (from: number, to: number) => (from === 0 ? "0%" : `${Math.round(((from - to) / from) * 100)}%`)
+
+console.log(`\n${rows.length} sampled variants, ${events.toLocaleString()} events (${values.file})\n`)
+console.log(` issues today ${currentGroups}`)
+console.log(` issues proposed ${candidateGroups} (${pct(currentGroups, candidateGroups)} fewer)`)
+
+// Parity: the TS mirror should group exactly as the MV hashed. A mismatch means
+// the replay is not measuring production, and its numbers are not to be trusted.
+const hashed = keyed.filter((k) => k.row.fingerprintHash !== undefined)
+if (hashed.length > 0) {
+ const hashes = distinct(hashed.map((k) => k.row.fingerprintHash ?? ""))
+ const mirrored = distinct(hashed.map((k) => k.current))
+ console.log(` production hashes ${hashes} (TS mirror: ${mirrored}${hashes === mirrored ? ", in parity" : ", NOT in parity"})`)
+}
+
+const groupBy = (items: ReadonlyArray, key: (k: Keyed) => K) => {
+ const map = new Map>()
+ for (const item of items) map.set(key(item), [...(map.get(key(item)) ?? []), item])
+ return map
+}
+
+const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n - 1)}…` : s)
+const oneLine = (s: string) => s.replace(/\s+/g, " ").trim()
+
+console.log(`\nWhere the change lands (service · type: issues today -> proposed)\n`)
+const byType = [...groupBy(keyed, (k) => `${k.row.serviceName} · ${k.row.exceptionType || "(no type)"}`)]
+ .map(([label, items]) => ({
+ label,
+ before: distinct(items.map((k) => k.current)),
+ after: distinct(items.map((k) => k.candidate)),
+ }))
+ .filter((entry) => entry.before !== entry.after)
+ .sort((a, b) => b.before - b.after - (a.before - a.after))
+for (const entry of byType.slice(0, top)) {
+ console.log(` ${String(entry.before).padStart(4)} -> ${String(entry.after).padEnd(4)} ${clip(entry.label, 90)}`)
+}
+if (byType.length === 0) console.log(" (no change)")
+
+// The merges are where a proposal goes wrong: read the messages and ask whether
+// they are really one bug.
+console.log(`\nLargest merges (check these are one bug)\n`)
+const merges = [...groupBy(keyed, (k) => k.candidate).values()]
+ .map((items) => ({ items, merged: distinct(items.map((k) => k.current)) }))
+ .filter((entry) => entry.merged > 1)
+ .sort((a, b) => b.merged - a.merged)
+for (const { items, merged } of merges.slice(0, top)) {
+ const head = items[0].row
+ console.log(` ${merged} issues -> 1 ${head.serviceName} · ${head.exceptionType || "(no type)"}`)
+ for (const message of [...new Set(items.map((k) => oneLine(k.row.statusMessage)))].slice(0, 3)) {
+ console.log(` ${clip(message || "(empty message)", 110)}`)
+ }
+}
+if (merges.length === 0) console.log(" (none)")
+
+const splits = [...groupBy(keyed, (k) => k.current).values()].filter(
+ (items) => distinct(items.map((k) => k.candidate)) > 1,
+)
+console.log(`\nIssues the proposal would split: ${splits.length}`)
+for (const items of splits.slice(0, top)) {
+ const head = items[0].row
+ console.log(` ${head.serviceName} · ${head.exceptionType} -> ${distinct(items.map((k) => k.candidate))}`)
+}
+console.log("")
diff --git a/scripts/errors-replay/candidate.ts b/scripts/errors-replay/candidate.ts
new file mode 100644
index 0000000000..e13cf2177e
--- /dev/null
+++ b/scripts/errors-replay/candidate.ts
@@ -0,0 +1,115 @@
+/**
+ * A PROPOSED fingerprint (v3), replayed against real samples before it ships.
+ *
+ * Nothing in production reads this. It exists so a grouping change can be
+ * measured (`bun run errors:replay`) instead of argued. When a proposal wins,
+ * port it to `packages/domain/src/tinybird/fingerprint.ts` and the MV SQL, and
+ * bump FINGERPRINT_VERSION.
+ *
+ * Each rule below targets a split seen in prod (2026-10-13 sample):
+ * 1. Caller frames: one upstream 503 became one issue per query that hit it,
+ * because frames 2-3 are the callers. A namespaced error type already names
+ * the failure, so only the throw site is kept for those.
+ * 2. Effect span frames (`(definition)`) vary the stack depth for one call path.
+ * 3. Filesystem paths outside the home dir (`/tmp/...`, `/srv/...`) and the
+ * random suffixes inside them. Route paths (`/api/...`) are still kept.
+ * 4. Prefixed and non-hex ids (`org_2Nn...`, `cus_...`, ULIDs) and UUIDs inside JSON.
+ */
+import { Option, Schema } from "effect"
+import {
+ applyRedactions,
+ computeFingerprintInputs,
+ FRAME_LINE_PATTERN,
+ FRAME_REDACTIONS,
+ MAX_FINGERPRINT_FRAMES,
+ MSG_SCAN_CHARS,
+ MSG_SIGNATURE_CHARS,
+ MSG_TEXT_REDACTIONS,
+ type Redactions,
+} from "../../packages/domain/src/tinybird/fingerprint"
+
+export interface ReplayEvent {
+ readonly serviceName: string
+ readonly exceptionType: string
+ readonly statusMessage: string
+ readonly exceptionStacktrace: string
+}
+
+const FRAME_LINE_RE = new RegExp(FRAME_LINE_PATTERN)
+
+/** `@maple/http/errors/Foo`, `effect/ai/AiError`: a type that names one failure. */
+const isNamespacedType = (type: string) => type.includes("/")
+
+const isEffectSpanFrame = (line: string) => line.includes("(definition)")
+
+const CANDIDATE_FRAME_REDACTIONS: Redactions = [
+ // pnpm/bun store paths carry the package version; a dependency bump is not a new bug.
+ ["@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*", "@#"],
+ ["\\(index [0-9]+\\)", "(index #)"],
+ ...FRAME_REDACTIONS,
+]
+
+const FS_ROOTS = "tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run"
+
+// Replacements may use `$1`; ClickHouse's replaceRegexpAll spells it `\1`. No
+// lookbehind: these must port to RE2 for the MV.
+const CANDIDATE_MSG_REDACTIONS: Redactions = [
+ // ClickHouse errors echo the failing query; the error class is the bug, the
+ // query text (an agent's own SQL, mostly) is a value.
+ ["(DB::Exception: [A-Za-z ]+?)[:.(][\\s\\S]*", "$1"],
+ ["^(\\[Error\\] [A-Za-z ]+?)[`'\"#:.(][\\s\\S]*", "$1"],
+ // Filesystem paths, rooted where a route never is, and only at the start of a
+ // token so `/api/data/x` is not read as `/data`. Runs before the home-dir pass.
+ [`(^|[\\s'"(\`=])(~|/(${FS_ROOTS}|Users/[^/ ]+|home/[^/ ]+))(/[^ '"\`:,)]*)?`, "$1PATH"],
+ // Prefixed ids: Clerk `org_`/`user_`, Stripe `cus_`/`sub_`, and their like.
+ ["\\b[a-z]{2,8}_[A-Za-z0-9]{10,}\\b", "ID"],
+ // ULIDs: 26 Crockford base32 characters.
+ ["\\b[0-9A-HJKMNP-TV-Z]{26}\\b", "ID"],
+ ...MSG_TEXT_REDACTIONS,
+]
+
+const CANDIDATE_JSON_VALUE_REDACTIONS: Redactions = [["[0-9a-fA-F-]{6,}|[0-9]+", "#"]]
+
+function candidateFrames(event: ReplayEvent): string {
+ const keep = isNamespacedType(event.exceptionType) ? 1 : MAX_FINGERPRINT_FRAMES
+ return event.exceptionStacktrace
+ .split("\n")
+ .filter((line) => FRAME_LINE_RE.test(line) && !isEffectSpanFrame(line))
+ .slice(0, keep)
+ .map((line) => applyRedactions(line, CANDIDATE_FRAME_REDACTIONS))
+ .join("\n")
+}
+
+function candidateMessage(message: string): string {
+ const json = parseJsonObject(message)
+ if (json !== undefined) {
+ return Object.keys(json)
+ .map((key) => `${key}=${applyRedactions(JSON.stringify(json[key]), CANDIDATE_JSON_VALUE_REDACTIONS)}`)
+ .sort()
+ .join("|")
+ }
+ return applyWithGroups(message.slice(0, MSG_SCAN_CHARS), CANDIDATE_MSG_REDACTIONS).slice(0, MSG_SIGNATURE_CHARS)
+}
+
+const applyWithGroups = (value: string, redactions: Redactions): string =>
+ redactions.reduce((acc, [pattern, replacement]) => acc.replace(new RegExp(pattern, "g"), replacement), value)
+
+const decodeJsonObject = Schema.decodeUnknownOption(
+ Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
+)
+
+const parseJsonObject = (text: string): Record | undefined =>
+ Option.getOrUndefined(decodeJsonObject(text))
+
+/** The grouping key production computes today (the TS mirror of the MV). */
+export function currentKey(event: ReplayEvent): string {
+ const inputs = computeFingerprintInputs(event)
+ return [event.serviceName, event.exceptionType, inputs.fpFrames, inputs.msgSignature].join("\u0000")
+}
+
+/** The proposed grouping key. */
+export function candidateKey(event: ReplayEvent): string {
+ return [event.serviceName, event.exceptionType, candidateFrames(event), candidateMessage(event.statusMessage)].join(
+ "\u0000",
+ )
+}
diff --git a/scripts/errors-replay/corpus.jsonl b/scripts/errors-replay/corpus.jsonl
new file mode 100644
index 0000000000..d541e6c844
--- /dev/null
+++ b/scripts/errors-replay/corpus.jsonl
@@ -0,0 +1,18 @@
+{"serviceName":"web","exceptionType":"@acme/http/errors/WarehouseUpstreamError","statusMessage":"HTTP 503","exceptionStacktrace":"@acme/http/errors/WarehouseUpstreamError: HTTP 503\n at Object.transform (https://app.example.com/assets/Schema-va6l_Bfn.js:2:58950)\n at e (https://app.example.com/assets/Schema-va6l_Bfn.js:2:32917)\n at serviceUsage (https://app.example.com/assets/query-atoms-l4GZq8Vn.js:1:6826)\n at QueryEngine.getServiceUsage (https://app.example.com/assets/query-atoms-l4GZq8Vn.js:1:151730)","count":5}
+{"serviceName":"web","exceptionType":"@acme/http/errors/WarehouseUpstreamError","statusMessage":"HTTP 503","exceptionStacktrace":"@acme/http/errors/WarehouseUpstreamError: HTTP 503\n at Object.transform (https://app.example.com/assets/Schema-va6l_Bfn.js:2:58950)\n at e (https://app.example.com/assets/Schema-va6l_Bfn.js:2:32917)\n at QueryEngine.getServiceUsage (https://app.example.com/assets/query-atoms-l4GZq8Vn.js:1:151730)\n at QueryEngine.getServiceUsage (definition) (https://app.example.com/assets/query-atoms-l4GZq8Vn.js:1:47457)","count":5}
+{"serviceName":"web","exceptionType":"@acme/http/errors/WarehouseUpstreamError","statusMessage":"HTTP 503","exceptionStacktrace":"@acme/http/errors/WarehouseUpstreamError: HTTP 503\n at Object.transform (https://app.example.com/assets/Schema-va6l_Bfn.js:2:58950)\n at e (https://app.example.com/assets/Schema-va6l_Bfn.js:2:32917)\n at QueryEngine.scopeServicesToNamespaces (https://app.example.com/assets/query-atoms-l4GZq8Vn.js:1:51345)","count":4}
+{"serviceName":"web","exceptionType":"@acme/http/errors/WarehouseUpstreamError","statusMessage":"HTTP 503","exceptionStacktrace":"@acme/http/errors/WarehouseUpstreamError: HTTP 503\n at Object.transform (https://app.example.com/assets/Schema-va6l_Bfn.js:2:58950)\n at e (https://app.example.com/assets/Schema-va6l_Bfn.js:2:32917)\n at serviceDetailOverview (https://app.example.com/assets/query-atoms-l4GZq8Vn.js:1:6826)","count":4}
+{"serviceName":"cli","exceptionType":"@acme/cli/LocalStoreDirtyError","statusMessage":"the local store at /tmp/acme-checkpoint.OXto4w/data was not cleanly closed. Run `acme restore --yes` to restore.","exceptionStacktrace":"","count":1}
+{"serviceName":"cli","exceptionType":"@acme/cli/LocalStoreDirtyError","statusMessage":"the local store at /srv/proxy/data/acme was not cleanly closed. Run `acme restore --yes` to restore.","exceptionStacktrace":"","count":1}
+{"serviceName":"cli","exceptionType":"@acme/cli/LocalStoreDirtyError","statusMessage":"the local store at ~/.acme/data was not cleanly closed. Run `acme restore --yes` to restore.","exceptionStacktrace":"","count":40}
+{"serviceName":"api","exceptionType":"@acme/backend/billing/CustomerNotFound","statusMessage":"No customer for org org_2NnXk9QbVt7Lm3Pq","exceptionStacktrace":"","count":3}
+{"serviceName":"api","exceptionType":"@acme/backend/billing/CustomerNotFound","statusMessage":"No customer for org org_2RbTy8WcZe1Hj6Ks","exceptionStacktrace":"","count":2}
+{"serviceName":"api","exceptionType":"@acme/backend/billing/CustomerNotFound","statusMessage":"No customer for org org_2KdPw4MnAs9Fg2Lr","exceptionStacktrace":"","count":1}
+{"serviceName":"api","exceptionType":"ProblemError","statusMessage":"{\"title\":\"Not found\",\"instance\":\"e29b41d4-a716-4466-5544-0000a3f19c2e\"}","exceptionStacktrace":"","count":2}
+{"serviceName":"api","exceptionType":"ProblemError","statusMessage":"{\"title\":\"Not found\",\"instance\":\"a3f1c2e4-b716-4466-9544-00007e2a1b0d\"}","exceptionStacktrace":"","count":2}
+{"serviceName":"api","exceptionType":"HttpClientError","statusMessage":"Transport error (POST /api/query-engine/service-overview)","exceptionStacktrace":"","count":9}
+{"serviceName":"api","exceptionType":"HttpClientError","statusMessage":"Transport error (POST /api/data/export)","exceptionStacktrace":"","count":4}
+{"serviceName":"api","exceptionType":"DatabaseError","statusMessage":"relation \"planetscale_events\" does not exist","exceptionStacktrace":"","count":6}
+{"serviceName":"api","exceptionType":"DatabaseError","statusMessage":"relation \"anomaly_detector_states\" does not exist","exceptionStacktrace":"","count":6}
+{"serviceName":"web","exceptionType":"TypeError","statusMessage":"Cannot read properties of undefined (reading 'id')","exceptionStacktrace":"TypeError: Cannot read properties of undefined (reading 'id')\n at renderRow (https://app.example.com/assets/list-a1b2c3d4.js:1:200)\n at map (https://app.example.com/assets/list-a1b2c3d4.js:1:900)\n at List (https://app.example.com/assets/list-a1b2c3d4.js:1:1200)","count":12}
+{"serviceName":"web","exceptionType":"TypeError","statusMessage":"Cannot read properties of undefined (reading 'id')","exceptionStacktrace":"TypeError: Cannot read properties of undefined (reading 'id')\n at renderRow (https://app.example.com/assets/list-a1b2c3d4.js:1:200)\n at map (https://app.example.com/assets/list-a1b2c3d4.js:1:900)\n at Sidebar (https://app.example.com/assets/sidebar-e5f6a7b8.js:1:330)","count":3}
diff --git a/scripts/errors-tick.ts b/scripts/errors-tick.ts
new file mode 100644
index 0000000000..4ab639ca58
--- /dev/null
+++ b/scripts/errors-tick.ts
@@ -0,0 +1,45 @@
+/**
+ * Fire the alerting Worker's minute cron once, on a local `bun dev` stack.
+ *
+ * That cron runs the errors tick (candidates -> issues, incidents, regressions)
+ * with fix verification, and also the alert and escalation lanes
+ * (`apps/alerting/src/schedule.ts`): alert rules evaluate and may notify too.
+ * It is the way to see newly ingested exceptions become issues without waiting.
+ *
+ * bun run errors:tick
+ * bun run errors:tick -- --url https://my-branch.alerting.localhost
+ *
+ * The Worker skips every cron off production unless MAPLE_ALERTING_ALLOW_NONPROD=1
+ * is in the env `bun dev` was started with; the alerting log says so once.
+ */
+import { parseArgs } from "node:util"
+
+const MINUTE_CRON = "* * * * *"
+
+const { values } = parseArgs({
+ options: {
+ url: { type: "string" },
+ cron: { type: "string", default: MINUTE_CRON },
+ },
+})
+
+const base = (values.url ?? process.env.MAPLE_DEV_ALERTING_URL ?? "https://alerting.localhost").replace(/\/$/, "")
+const cron = values.cron ?? MINUTE_CRON
+const target = `${base}/cdn-cgi/handler/scheduled?cron=${encodeURIComponent(cron)}`
+
+const started = Date.now()
+const response = await fetch(target).catch((cause: unknown) => {
+ console.error(`Could not reach ${base}: ${String(cause)}`)
+ console.error("Is the stack up with alerting in it? `bun dev api web ingest alerting`")
+ process.exit(1)
+})
+const body = (await response.text()).trim()
+
+if (!response.ok) {
+ console.error(`${response.status} from ${target}${body ? `\n${body}` : ""}`)
+ process.exit(1)
+}
+
+console.log(`Fired "${cron}" on ${base} in ${Date.now() - started}ms${body ? ` (${body})` : ""}.`)
+console.log("Issues appear once the tick finishes; the alerting log shows the run.")
+console.log("Nothing happened? Start `bun dev` with MAPLE_ALERTING_ALLOW_NONPROD=1.")