diff --git a/apps/chat-bot/package.json b/apps/chat-bot/package.json index b056fe559d..c436ddfa51 100644 --- a/apps/chat-bot/package.json +++ b/apps/chat-bot/package.json @@ -7,7 +7,6 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@maple-dev/effect-sdk": "workspace:*", "@maple/backend": "workspace:*", "@maple/chat-platform": "workspace:*", "@maple/domain": "workspace:*", diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 1952589f39..1988b1a4ae 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -368,6 +368,19 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "560a5a62db3961c68d815eeab700525cb3a59df3a2aadcf84033417c76a87f0b", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + // TODO(v30): what changed, whether any part is rewritten or any row + // moves, and what this edge does NOT backfill. + // + // projectRevision is carried forward deliberately: it is a hardcoded + // constant that no longer tracks the generator's header, and the identity + // this gate compares is the fingerprint/digest pair. + version: 30, + fingerprint: "5e8afdb803653665", + digest: "5e8afdb80365366551f730f44b42482c16d8f8923ba27060600ce4df1beec44a", + manifestDigest: "7fe927d0537e697ee7e9ad0c565c3294efee9e958d9894292afed234fbe560ab", + projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", + }), ] as const) /** Immutable SQLite control DDL identities, checked by clickhouse:schema:check. */ diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 11595d373e..c73bd20890 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,7 +1,7 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 29 as const +export const LOCAL_SCHEMA_VERSION = 30 as const /** SQLite eventing state has its own independent version sequence. */ export const LOCAL_CONTROL_SCHEMA_VERSION = 1 as const diff --git a/apps/cli/src/server/local-store-migrations/steps.ts b/apps/cli/src/server/local-store-migrations/steps.ts index 77060baafb..9b9fc0d6fb 100644 --- a/apps/cli/src/server/local-store-migrations/steps.ts +++ b/apps/cli/src/server/local-store-migrations/steps.ts @@ -1223,5 +1223,35 @@ export const LOCAL_STORE_STEPS: ReadonlyArray = [ }, ], }, + { + // TODO(v29 -> v30): what changes, and what is NOT backfilled. Fill beforeBootstrap + // with the ADD COLUMN / view drops an IF NOT EXISTS bootstrap cannot do, then the + // plan line and dispositions. The v30 physical verify fails an unfinished row. + id: "local-0029-to-0030-error-fingerprint-v3", + from: 29, + to: 30, + description: "Rebuild the error-events views with fingerprint v3 grouping and a version-salted hash", + clonedBefore: "the views are replaced", + beforeBootstrap: [dropTables("error_events_mv", "error_events_by_time_mv")], + plan: [ + [ + "rebuild-error-events-views", + "Drop and recreate the error-events views so new occurrences are hashed with fingerprint v3", + ], + ], + verifies: "Verify the v30 physical schema and retained raw telemetry counts", + dispositions: [ + TRACES_UNDER_REPLACED_VIEWS, + { + name: "error_events", + classification: "derived", + disposition: "preserve-exact", + guarantee: + "Existing rows keep their v2 FingerprintHash untouched; occurrences materialized after the migration carry v3 hashes, and the v2 rows age out with the retention window.", + ...ERROR_RETENTION, + }, + { ...ERROR_EVENTS_BY_TIME_FORWARD_ONLY, disposition: "preserve-exact" }, + ], + }, // local-schema:bump appends the next step above this line. ] diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 375ca56c20..330a2acb56 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -28,6 +28,7 @@ import schemaV26Sql from "./schema/local-schema-v26.sql" with { type: "text" } import schemaV27Sql from "./schema/local-schema-v27.sql" with { type: "text" } import schemaV28Sql from "./schema/local-schema-v28.sql" with { type: "text" } import schemaV29Sql from "./schema/local-schema-v29.sql" with { type: "text" } +import schemaV30Sql from "./schema/local-schema-v30.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -101,6 +102,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV27Sql, schemaV28Sql, schemaV29Sql, + schemaV30Sql, ] export interface LocalSchemaSnapshot { @@ -187,6 +189,7 @@ export const LOCAL_SCHEMA_V26 = localSchemaIdentity(26) export const LOCAL_SCHEMA_V27 = localSchemaIdentity(27) export const LOCAL_SCHEMA_V28 = localSchemaIdentity(28) export const LOCAL_SCHEMA_V29 = localSchemaIdentity(29) +export const LOCAL_SCHEMA_V30 = localSchemaIdentity(30) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 787e0835e0..457820f972 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "6946cd8703d07c0be3ef010ccbde01194cbfd3ac8e324dd6f36da99132a6fbba", + "projectRevision": "a9e17246b1cf0b92e8af1cb6a16606c9de70698233d82bb295231da6e22da702", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v30.sql b/apps/cli/src/server/schema/local-schema-v30.sql new file mode 100644 index 0000000000..bd92e30564 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v30.sql @@ -0,0 +1,2148 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: a9e17246b1cf0b92e8af1cb6a16606c9de70698233d82bb295231da6e22da702 +-- localSchemaVersion: 30 + +CREATE TABLE IF NOT EXISTS ai_crawler_requests ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + ServiceName LowCardinality(String), + Crawler LowCardinality(String), + Host LowCardinality(String), + Path String, + HttpStatus UInt16 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + Model LowCardinality(String), + AgentName LowCardinality(String), + ToolName LowCardinality(String), + SpanId String, + ParentSpanId String, + Duration UInt64, + IsError UInt8, + IsLlmCall UInt8, + IsToolCall UInt8, + Tokens Float64, + Cost Float64, + ResponseId String, + VendorVersion LowCardinality(String), + InputTokens Float64, + CacheReadTokens Float64, + CacheWriteTokens Float64, + OutputTokens Float64, + ReasoningTokens Float64, + ErrorType LowCardinality(String), + StatusMessage String, + ToolDescription String, + FailedToolCallResult String, + ErrorFingerprint UInt64 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String), + SkipReason LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + SpanKind LowCardinality(String) DEFAULT '', + IsRoot UInt8 DEFAULT 0, + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_facets_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName String, + HttpMethod LowCardinality(String), + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + HasError UInt8, + TraceCount SimpleAggregateFunction(sum, UInt64), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError) +TTL Hour + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_crawler_requests_mv TO ai_crawler_requests AS +SELECT + OrgId, + Timestamp, + TraceId, + ServiceName, + arrayElement(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'Meta-ExternalAgent', 'Meta-WebIndexer', 'Meta-ExternalFetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'])) AS Crawler, + lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')) AS Host, + leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path, + toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus + FROM traces + WHERE SpanKind = 'Server' + AND multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai']) > 0 + AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + SpanAttributes['maple_ai.model'] AS Model, + SpanAttributes['maple_ai.agent.name'] AS AgentName, + SpanAttributes['maple_ai.tool.name'] AS ToolName, + SpanId, + ParentSpanId, + Duration, + toUInt8(SpanAttributes['maple_ai.error'] = '1') AS IsError, + toUInt8(SpanAttributes['maple_ai.llm_call'] = '1') AS IsLlmCall, + toUInt8(SpanAttributes['maple_ai.tool_call'] = '1') AS IsToolCall, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS Tokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cost']) AS Cost, + SpanAttributes['maple_ai.response.id'] AS ResponseId, + SpanAttributes['maple_ai.vendor.version'] AS VendorVersion, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) AS InputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) AS CacheReadTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) AS CacheWriteTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) AS OutputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS ReasoningTokens, + SpanAttributes['error.type'] AS ErrorType, + leftUTF8(StatusMessage, 400) AS StatusMessage, + SpanAttributes['maple_ai.tool.description'] AS ToolDescription, + SpanAttributes['maple_ai.tool.error_result'] AS FailedToolCallResult, + if(SpanAttributes['maple_ai.error'] = '1', cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(leftUTF8(coalesce(nullIf(SpanAttributes['maple_ai.tool.error_result'], ''), leftUTF8(StatusMessage, 400)), 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#')), 0) AS ErrorFingerprint + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\['), + splitByChar('\n', _exStack) + ), + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\['), + splitByChar('\n', _exStack) + ), + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + left(_msgText, multiIf( + position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1, + position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1, + position(_msgText, '\n') > 3, toInt64(position(_msgText, '\n')) - 1, + least(toInt64(length(_msgText)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + SpanKind, + toUInt8(ParentSpanId = '') AS IsRoot + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_facets_hourly_mv TO trace_facets_hourly AS +SELECT + OrgId, + toStartOfHour(Timestamp) AS Hour, + ServiceName, + SpanName, + HttpMethod, + HttpStatusCode, + DeploymentEnv, + ServiceNamespace, + HasError, + count() AS TraceCount, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM trace_list_mv + GROUP BY OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index e1bf537c9f..bd92e30564 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 6946cd8703d07c0be3ef010ccbde01194cbfd3ac8e324dd6f36da99132a6fbba --- localSchemaVersion: 29 +-- projectRevision: a9e17246b1cf0b92e8af1cb6a16606c9de70698233d82bb295231da6e22da702 +-- localSchemaVersion: 30 CREATE TABLE IF NOT EXISTS ai_crawler_requests ( OrgId LowCardinality(String), @@ -1039,37 +1039,6 @@ WITH if(_useAttrs, SpanAttributes['exception.stacktrace'], '') ) AS _exStack, if(_useAttrs, _exMsg, StatusMessage) AS _msgText, - -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old - -- rule accepted any line with a colon-digit, which let non-frame lines - -- in: Drizzle's `params: ` line, and the `Type: message` - -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values - -- and message text then entered the hash and split one bug into - -- thousands of issues — 23,035 fingerprints for six real - -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError - -- ones. - -- - -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, - -- as is every redaction below. They used to be hand-copied here, which - -- let the reference implementation the tests exercise drift away from - -- the SQL that actually runs, silently. - arraySlice( - arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), - splitByChar('\n', _exStack) - ), - 1, 3 - ) AS _rawFrames, - -- Redact every volatile token a frame line can carry: the URL origin - -- (so preview hosts share one fingerprint), Vite's 8-char bundle - -- content hash (so a deploy does not re-split every triaged browser and - -- Worker issue), then line numbers, hex pointers and long id runs. See - -- FRAME_REDACTIONS for the order and the reasoning. - arrayMap( - line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), - _rawFrames - ) AS _topFrames, - if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, - arrayStringConcat(_topFrames, '\n') AS _fpFrames, -- JSON detection for the message signature below. isValidJSON(_msgText) AS _isJson, _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, @@ -1081,7 +1050,7 @@ WITH arrayStringConcat( arraySort( arrayMap( - kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')), JSONExtractKeysAndValuesRaw(_msgText) ) ), @@ -1095,15 +1064,50 @@ WITH -- restores that discrimination, and it cannot reinflate cardinality the -- way a raw prefix would because everything variable is redacted first: -- emails, URL origins, home directories, query strings, quoted values, - -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, -- what is deliberately kept, and the one residual it cannot reach. multiIf( _isJsonObj, _jsonSig, substringUTF8( - replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), 1, 120 ) ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\['), + splitByChar('\n', _exStack) + ), + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, -- Display-only, best-effort human label (decoupled from the fingerprint: -- many labels may map to one hash). The broad key list here is a DISPLAY -- heuristic only; the fingerprint above makes no key-name assumption. @@ -1154,7 +1158,7 @@ WITH _exMsg AS ExceptionMessage, _exStack AS ExceptionStacktrace, _topFrame AS TopFrame, - cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash, StatusMessage, Duration, _errorLabel AS ErrorLabel, @@ -1196,37 +1200,6 @@ WITH if(_useAttrs, SpanAttributes['exception.stacktrace'], '') ) AS _exStack, if(_useAttrs, _exMsg, StatusMessage) AS _msgText, - -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old - -- rule accepted any line with a colon-digit, which let non-frame lines - -- in: Drizzle's `params: ` line, and the `Type: message` - -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values - -- and message text then entered the hash and split one bug into - -- thousands of issues — 23,035 fingerprints for six real - -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError - -- ones. - -- - -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, - -- as is every redaction below. They used to be hand-copied here, which - -- let the reference implementation the tests exercise drift away from - -- the SQL that actually runs, silently. - arraySlice( - arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), - splitByChar('\n', _exStack) - ), - 1, 3 - ) AS _rawFrames, - -- Redact every volatile token a frame line can carry: the URL origin - -- (so preview hosts share one fingerprint), Vite's 8-char bundle - -- content hash (so a deploy does not re-split every triaged browser and - -- Worker issue), then line numbers, hex pointers and long id runs. See - -- FRAME_REDACTIONS for the order and the reasoning. - arrayMap( - line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), - _rawFrames - ) AS _topFrames, - if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, - arrayStringConcat(_topFrames, '\n') AS _fpFrames, -- JSON detection for the message signature below. isValidJSON(_msgText) AS _isJson, _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, @@ -1238,7 +1211,7 @@ WITH arrayStringConcat( arraySort( arrayMap( - kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')), JSONExtractKeysAndValuesRaw(_msgText) ) ), @@ -1252,15 +1225,50 @@ WITH -- restores that discrimination, and it cannot reinflate cardinality the -- way a raw prefix would because everything variable is redacted first: -- emails, URL origins, home directories, query strings, quoted values, - -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, -- what is deliberately kept, and the one residual it cannot reach. multiIf( _isJsonObj, _jsonSig, substringUTF8( - replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), 1, 120 ) ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\['), + splitByChar('\n', _exStack) + ), + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, -- Display-only, best-effort human label (decoupled from the fingerprint: -- many labels may map to one hash). The broad key list here is a DISPLAY -- heuristic only; the fingerprint above makes no key-name assumption. @@ -1311,7 +1319,7 @@ WITH _exMsg AS ExceptionMessage, _exStack AS ExceptionStacktrace, _topFrame AS TopFrame, - cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash, StatusMessage, Duration, _errorLabel AS ErrorLabel, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index c420529918..edbe497b39 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -33,6 +33,7 @@ import { LOCAL_SCHEMA_V27, LOCAL_SCHEMA_V28, LOCAL_SCHEMA_V29, + LOCAL_SCHEMA_V30, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -97,11 +98,11 @@ const [v10ToV11ProductEventsModule] = localStoreMigrations.filter( ) describe("current local schema identity", () => { - it("matches the generated v29 revision and keeps the issue-297 identity frozen", () => { + it("matches the generated v30 revision and keeps the issue-297 identity frozen", () => { // oxlint-disable-next-line test-slop/no-constant-pin -- rewritten by local-schema:bump; catches an unbumped schema edit - expect(SCHEMA_FINGERPRINT).toBe("c0c78cf931a7f897") + expect(SCHEMA_FINGERPRINT).toBe("5e8afdb803653665") // oxlint-disable-next-line test-slop/no-constant-pin -- rewritten by local-schema:bump; catches an unbumped schema edit - expect(SCHEMA_DIGEST).toBe("c0c78cf931a7f897f41f8102841bf33dc320b6a2c852eaf78ffd58a375f8f3b5") + expect(SCHEMA_DIGEST).toBe("5e8afdb80365366551f730f44b42482c16d8f8923ba27060600ce4df1beec44a") // oxlint-disable-next-line test-slop/no-constant-pin -- frozen identity that stores already on disk carry expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", @@ -109,8 +110,8 @@ describe("current local schema identity", () => { expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) // oxlint-disable-next-line test-slop/no-constant-pin -- anchor that local-schema:bump rewrites on every bump - expect(CURRENT_LOCAL_SCHEMA.version).toBe(29) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V29) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(30) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V30) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -382,7 +383,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 30, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 31, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -530,6 +531,7 @@ describe("local migration registry", () => { "local-0026-to-0027-alert-checks-skip-reason", "local-0027-to-0028-service-overview-spans-span-kind", "local-0028-to-0029-external-edges-legacy-db-system", + "local-0029-to-0030-error-fingerprint-v3", ] const versions = (id: string) => { const match = /^local-(\d{4})-to-(\d{4})-[a-z0-9]+(-[a-z0-9]+)*$/.exec(id) @@ -1510,6 +1512,7 @@ describe("v10 -> v11 product events module", () => { "local-0026-to-0027-alert-checks-skip-reason", "local-0027-to-0028-service-overview-spans-span-kind", "local-0028-to-0029-external-edges-legacy-db-system", + "local-0029-to-0030-error-fingerprint-v3", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 9b42c4672c..87c794d03a 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 29 and .schema == "c0c78cf931a7f897"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 30 and .schema == "5e8afdb803653665"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index be640ab52e..d6482c4235 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "6946cd8703d07c0be3ef010ccbde01194cbfd3ac8e324dd6f36da99132a6fbba"; +pub const PROJECT_REVISION: &str = "a9e17246b1cf0b92e8af1cb6a16606c9de70698233d82bb295231da6e22da702"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/web/src/components/infra/cloudflare/cloudflare-worker-table.tsx b/apps/web/src/components/infra/cloudflare/cloudflare-worker-table.tsx index 0127596c1b..b239d39f8e 100644 --- a/apps/web/src/components/infra/cloudflare/cloudflare-worker-table.tsx +++ b/apps/web/src/components/infra/cloudflare/cloudflare-worker-table.tsx @@ -1,4 +1,3 @@ -import { Skeleton } from "@maple/ui/components/ui/skeleton" import { cn } from "@maple/ui/lib/utils" import { errorRateClass } from "@maple/ui/lib/error-rate" import { EMPTY_VALUE, formatErrorRate, formatLatency, formatNumber } from "@maple/ui/lib/format" @@ -38,27 +37,6 @@ function WorkerColumns({ showAccount }: { showAccount: boolean }) { ) } -export function CloudflareWorkerTableLoading() { - return ( - - - - - - -
- - -
- - - - - - - ) -} - /** Rows arrive already ordered (problems first); the table doesn't re-sort them. */ export function CloudflareWorkerTable({ rows, diff --git a/bun.lock b/bun.lock index 2edd74c1b7..101654e021 100644 --- a/bun.lock +++ b/bun.lock @@ -106,7 +106,6 @@ "apps/chat-bot": { "name": "@maple/chat-bot", "dependencies": { - "@maple-dev/effect-sdk": "workspace:*", "@maple/backend": "workspace:*", "@maple/chat-platform": "workspace:*", "@maple/domain": "workspace:*", diff --git a/packages/backend/src/services/errors/ErrorsService.test.ts b/packages/backend/src/services/errors/ErrorsService.test.ts index 8b82abf013..af4384c9c2 100644 --- a/packages/backend/src/services/errors/ErrorsService.test.ts +++ b/packages/backend/src/services/errors/ErrorsService.test.ts @@ -20,6 +20,7 @@ import { ErrorIssueEventId, ErrorIssueId, } from "@maple/domain/primitives" +import { FINGERPRINT_VERSION } from "@maple/domain/tinybird/fingerprint" import * as PG from "@maple-dev/effect-orm/postgres" import { AlertDestinations, @@ -909,6 +910,20 @@ const scanRow = (overrides: Record = {}): Record + Effect.gen(function* () { + const database = yield* Database + const rows = yield* database.execute((db) => + db.run( + PG.from(ErrorIssues) + .select() + .where(($) => [$.id.eq(issueId)]), + ), + ) + assert.lengthOf(rows, 1) + return rows[0]! + }) + const loadIssuesByFingerprint = (fingerprintHash: string) => Effect.gen(function* () { const database = yield* Database @@ -2437,6 +2452,95 @@ describe("ErrorsService.runTick", () => { assert.lengthOf(purgedStates, 0) }).pipe(Effect.provide(makeErrorsLayer())), ) + + it.effect("the retention phase archives an issue left on an older fingerprint version", () => + Effect.gen(function* () { + const errors = yield* ErrorsService + yield* TestClock.setTime(RETENTION_TICK_MS) + const stale = asIssueId(randomUUID()) + const current = asIssueId(randomUUID()) + yield* seedIssue(stale, { fingerprintVersion: FINGERPRINT_VERSION - 1 }) + yield* seedIssue(current, { fingerprintVersion: FINGERPRINT_VERSION }) + + const result = yield* errors.runTick() + + assert.strictEqual(result.issuesArchived, 1) + assert.strictEqual((yield* loadIssueById(stale)).archivedAt, RETENTION_TICK_MS) + assert.isNull((yield* loadIssueById(current)).archivedAt) + }).pipe(Effect.provide(makeErrorsLayer())), + ) + + it.effect("an archived open issue that occurs again comes back with its state intact", () => { + const rows = [scanRow()] + return Effect.gen(function* () { + const errors = yield* ErrorsService + yield* TestClock.setTime(TICK_MS) + // Archived by a fingerprint bump while its old hash was still live: a + // self-hosted warehouse that has not applied the new view, or a deploy gap. + const issueId = asIssueId(randomUUID()) + yield* seedIssue(issueId, { + fingerprintHash: SCAN_FINGERPRINT, + fingerprintVersion: FINGERPRINT_VERSION - 1, + workflowState: "in_progress", + priority: 1, + archivedAt: TICK_MS - DAY_MS, + }) + + yield* errors.runTick() + + const issue = yield* loadIssueById(issueId) + assert.isNull(issue.archivedAt) + assert.strictEqual(issue.workflowState, "in_progress") + assert.strictEqual(issue.priority, 1) + // Restamped, so the version sweep does not archive it again. + assert.strictEqual(issue.fingerprintVersion, FINGERPRINT_VERSION) + }).pipe(Effect.provide(makeErrorsLayer(() => rows))) + }) + + it.effect("a regression on an archived resolved issue reopens it where it can be seen", () => { + const rows = [scanRow()] + return Effect.gen(function* () { + const errors = yield* ErrorsService + yield* TestClock.setTime(TICK_MS) + const issueId = asIssueId(randomUUID()) + yield* seedIssue(issueId, { + fingerprintHash: SCAN_FINGERPRINT, + fingerprintVersion: FINGERPRINT_VERSION, + workflowState: "done", + resolvedAt: TICK_MS - 20 * DAY_MS, + archivedAt: TICK_MS - 6 * DAY_MS, + }) + + yield* errors.runTick() + + const issue = yield* loadIssueById(issueId) + assert.strictEqual(issue.workflowState, "regressed") + assert.isNull(issue.archivedAt) + }).pipe(Effect.provide(makeErrorsLayer(() => rows))) + }) + + it.effect("an archived resolved issue stays archived for an occurrence from a pre-fix build", () => { + const rows = [scanRow({ serviceVersions: ["1.4.0"] })] + return Effect.gen(function* () { + const errors = yield* ErrorsService + yield* TestClock.setTime(TICK_MS) + const issueId = asIssueId(randomUUID()) + yield* seedIssue(issueId, { + fingerprintHash: SCAN_FINGERPRINT, + fingerprintVersion: FINGERPRINT_VERSION, + workflowState: "done", + resolvedAt: TICK_MS - 20 * DAY_MS, + resolvedVersionsJson: ["1.4.0"], + archivedAt: TICK_MS - 6 * DAY_MS, + }) + + yield* errors.runTick() + + const issue = yield* loadIssueById(issueId) + assert.strictEqual(issue.workflowState, "done") + assert.strictEqual(issue.archivedAt, TICK_MS - 6 * DAY_MS) + }).pipe(Effect.provide(makeErrorsLayer(() => rows))) + }) }) // Idle-cursor harness diff --git a/packages/backend/src/services/errors/error-tick-fingerprint.clickhouse.e2e.test.ts b/packages/backend/src/services/errors/error-tick-fingerprint.clickhouse.e2e.test.ts new file mode 100644 index 0000000000..f2c4c7249b --- /dev/null +++ b/packages/backend/src/services/errors/error-tick-fingerprint.clickhouse.e2e.test.ts @@ -0,0 +1,334 @@ +// SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. +// The fingerprint end to end: spans go into a real ClickHouse, its error views +// hash them, and the real errors tick turns the hashes into Postgres issues. + +import { randomUUID } from "node:crypto" +import { afterAll, afterEach, assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { FetchHttpClient } from "effect/http" +import { migrations } from "@maple/domain/clickhouse" +import { OrgId } from "@maple/domain/http" +import { ErrorIssueId } from "@maple/domain/primitives" +import { FINGERPRINT_VERSION } from "@maple/domain/tinybird/fingerprint" +import * as PG from "@maple-dev/effect-orm/postgres" +import { ErrorIssues } from "@maple/db/tables" +import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { Env } from "@maple/backend/platform/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" +import { NotificationDispatcher } from "@maple/backend/services/alerts/NotificationDispatcher" +import { TinybirdOrgTokenService } from "@maple/backend/services/integrations/TinybirdOrgTokenService" +import { OrgClickHouseSettingsService } from "@maple/backend/services/org/OrgClickHouseSettingsService" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { + applyRealMigrations, + clickhouseE2eEnabled, + clickhouseExec, + clickhousePassword, + clickhouseUrl, + clickhouseUser, + uniqueDatabase, +} from "@maple/backend/services/warehouse/clickhouse-e2e-support" +import { ErrorActorsService } from "./ErrorActorsService" +import { ErrorIssueWorkflowService } from "./ErrorIssueWorkflowService" +import { ErrorPolicyService } from "./ErrorPolicyService" +import { ErrorsService } from "./ErrorsService" +import { InvestigationService } from "./InvestigationService" + +const ORG = Schema.decodeSync(OrgId)("org_error_tick_fingerprint_e2e") +const asIssueId = Schema.decodeSync(ErrorIssueId) +const SERVICE = "api" +const createdDbs: TestDb[] = [] +const createdDatabases: string[] = [] + +afterEach(() => cleanupTestDbs(createdDbs)) +afterAll(async () => { + for (const name of createdDatabases) await clickhouseExec(`DROP DATABASE IF EXISTS ${name}`) +}) + +const quote = (value: string): string => `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` + +interface Span { + readonly type: string + readonly message: string + readonly stacktrace: string +} + +const frames = (...lines: ReadonlyArray) => ["Error: boom", ...lines].join("\n") + +/** + * Inside the tick's bootstrap window: the cutoff trails now by the ingestion lag + * (1-2 minutes) and the window opens 2 minutes before that. + */ +const spanTimestamp = () => + new Date(Date.now() - 150_000).toISOString().replace("T", " ").replace("Z", "").slice(0, 23) + +/** Three of each, so every fingerprint clears the promotion threshold. */ +const insertSpans = async (database: string, spans: ReadonlyArray): Promise => { + const ts = spanTimestamp() + const rows = spans.flatMap((span) => + [1, 2, 3].map((n) => { + const spanId = randomUUID() + return `(${quote(ORG)}, ${quote(ts)}, ${quote(`trace-${spanId}`)}, ${quote(spanId)}, '', 'GET /', 'Server', ${quote(SERVICE)}, 1000000, 'Error', ${quote(span.message)}, map(), map('service.version', ${quote(`1.0.${n}`)}), [toDateTime64(${quote(ts)}, 9)], ['exception'], [map('exception.type', ${quote(span.type)}, 'exception.message', ${quote(span.message)}, 'exception.stacktrace', ${quote(span.stacktrace)})])` + }), + ) + await clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, StatusMessage, SpanAttributes, ResourceAttributes, EventsTimestamp, EventsName, EventsAttributes) + VALUES\n${rows.join(",\n")}`, + database, + ) +} + +const materializedHashes = async (database: string): Promise> => + ( + await clickhouseExec( + `SELECT DISTINCT toString(FingerprintHash) FROM error_events WHERE OrgId = ${quote(ORG)} ORDER BY 1 FORMAT TabSeparated`, + database, + ) + ) + .split("\n") + .filter((line) => line !== "") + +const prepareWarehouse = async (): Promise => { + const database = uniqueDatabase("maple_error_tick_fp_e2e") + createdDatabases.push(database) + await clickhouseExec(`CREATE DATABASE IF NOT EXISTS ${database}`) + await applyRealMigrations(database) + return database +} + +/** The tick's real service graph over one ClickHouse database and one test Postgres. */ +const makeTickLayer = (database: string) => { + const testDb = createTestDb(createdDbs) + const databaseLive = testDb.layer + const envLive = Env.layer.pipe( + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "http://127.0.0.1:7181", + TINYBIRD_TOKEN: "unused-for-vanilla-clickhouse", + CLICKHOUSE_URL: clickhouseUrl, + CLICKHOUSE_PROVIDER: "clickhouse", + CLICKHOUSE_USER: clickhouseUser, + CLICKHOUSE_PASSWORD: clickhousePassword, + CLICKHOUSE_DATABASE: database, + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: ORG, + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 9).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "error-tick-e2e-lookup-key", + MAPLE_INGEST_PUBLIC_URL: "http://127.0.0.1:3474", + MAPLE_APP_BASE_URL: "http://127.0.0.1:3471", + }), + ), + ), + ) + const edgeCacheLive = EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive)) + const warehouseLive = Layer.effect(WarehouseQueryService, WarehouseQueryService.make).pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide( + Layer.mergeAll( + envLive, + OrgClickHouseSettingsService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, databaseLive, edgeCacheLive)), + ), + TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)), + ), + ), + ) + const errorActorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) + const errorIssueWorkflowLive = Layer.effect( + ErrorIssueWorkflowService, + ErrorIssueWorkflowService.make, + ).pipe( + Layer.provide(AuditLogService.layerMemory), + Layer.provide(databaseLive), + Layer.provide(errorActorsLive), + ) + return Layer.effect(ErrorsService, ErrorsService.make).pipe( + Layer.provide(envLive), + Layer.provide(databaseLive), + Layer.provide(warehouseLive), + Layer.provide(edgeCacheLive), + Layer.provide( + Layer.succeed(NotificationDispatcher, { + dispatch: () => Effect.succeed({ delivered: 0, failed: 0 }), + }), + ), + Layer.provide(errorActorsLive), + Layer.provide(errorIssueWorkflowLive), + Layer.provide(ErrorPolicyService.layer.pipe(Layer.provide(databaseLive))), + Layer.provide(InvestigationService.layer.pipe(Layer.provide(Layer.mergeAll(envLive, databaseLive)))), + Layer.provideMerge(databaseLive), + ) +} + +/** + * An org with issue state is always scanned. Discovery itself is pinned to the + * managed warehouse, which this harness does not run. + */ +const PLACEHOLDER_HASH = "placeholder" +const seedOrgState = Effect.gen(function* () { + const database = yield* Database + const now = Date.now() + yield* database.execute((db) => + db.run( + PG.insertInto(ErrorIssues).values({ + id: asIssueId(randomUUID()), + orgId: ORG, + fingerprintHash: PLACEHOLDER_HASH, + fingerprintVersion: FINGERPRINT_VERSION, + serviceName: SERVICE, + exceptionType: "Placeholder", + exceptionMessage: "", + topFrame: "", + firstSeenAt: now, + lastSeenAt: now, + createdAt: now, + updatedAt: now, + }), + ), + ) +}) + +const loadIssues = Effect.gen(function* () { + const database = yield* Database + return yield* database.execute((db) => + db.run( + PG.from(ErrorIssues) + .select() + .where(($) => [$.orgId.eq(ORG), $.fingerprintHash.neq(PLACEHOLDER_HASH)]), + ), + ) +}) + +describe.skipIf(!clickhouseE2eEnabled)("errors tick over the v3 fingerprint (ClickHouse e2e)", () => { + it.live( + "groups real spans into the issues v3 intends, stamped with the current version", + () => + Effect.gen(function* () { + const database = yield* Effect.promise(prepareWarehouse) + const upstream = (caller: string, status: number): Span => ({ + type: "@maple/http/errors/UpstreamUnavailable", + message: `upstream returned ${status}`, + stacktrace: frames( + " at toUpstreamError (/app/src/upstream.ts:40:11)", + ` at ${caller} (/app/src/${caller}.ts:12:3)`, + ), + }) + const echo = (query: string): Span => ({ + type: "", + message: `[Error] Unknown identifier \`ServiceVersion\` in scope ${query}`, + stacktrace: "", + }) + yield* Effect.promise(() => + insertSpans(database, [ + // One outage, two callers, two 5xx codes: one issue. + upstream("listTraces", 502), + upstream("serviceMap", 524), + // One missing column, two queries: one issue. + echo("SELECT ServiceVersion FROM traces"), + echo("SELECT * FROM (SELECT ServiceVersion FROM logs)"), + // Two different client errors: two issues. + upstream("listTraces", 401), + upstream("listTraces", 404), + ]), + ) + const hashes = yield* Effect.promise(() => materializedHashes(database)) + assert.lengthOf(hashes, 4) + + return yield* Effect.gen(function* () { + yield* seedOrgState + const errors = yield* ErrorsService + yield* errors.runTick() + + const issues = yield* loadIssues + assert.sameMembers( + issues.map((issue) => issue.fingerprintHash), + [...hashes], + ) + for (const issue of issues) { + assert.strictEqual(issue.fingerprintVersion, FINGERPRINT_VERSION) + assert.isNull(issue.archivedAt) + } + // The two merged issues carry both spans' occurrences; 401 and 404 their own. + assert.deepStrictEqual( + issues.map((issue) => issue.occurrenceCount).sort((a, b) => a - b), + [3, 3, 6, 6], + ) + }).pipe(Effect.provide(makeTickLayer(database))) + }), + 120_000, + ) + + it.live( + "brings back an archived issue a lagging warehouse still hashes the old way", + () => + Effect.gen(function* () { + // A self-hosted warehouse that has not applied migration 0039 yet: put + // the v2 views (0030) back over the v3 ones. + const database = yield* Effect.promise(prepareWarehouse) + const v2 = migrations.find((migration) => migration.version === 30) + assert.isDefined(v2) + for (const statement of v2.statements) { + assert.isString(statement) + yield* Effect.promise(() => clickhouseExec(statement, database)) + } + yield* Effect.promise(() => + insertSpans(database, [ + { + type: "DatabaseError", + message: "connection reset by peer", + stacktrace: frames(" at query (/app/src/db.ts:5:1)"), + }, + ]), + ) + const [v2Hash] = yield* Effect.promise(() => materializedHashes(database)) + assert.isString(v2Hash) + + return yield* Effect.gen(function* () { + // Archived by the version sweep while its old hash was still live. + const issueId = asIssueId(randomUUID()) + const db = yield* Database + const archivedAt = Date.now() - 60 * 60_000 + yield* db.execute((tx) => + tx.run( + PG.insertInto(ErrorIssues).values({ + id: issueId, + orgId: ORG, + fingerprintHash: v2Hash, + fingerprintVersion: FINGERPRINT_VERSION - 1, + serviceName: SERVICE, + exceptionType: "DatabaseError", + exceptionMessage: "connection reset by peer", + topFrame: "", + workflowState: "in_progress", + priority: 1, + occurrenceCount: 40, + firstSeenAt: archivedAt - 86_400_000, + lastSeenAt: archivedAt, + archivedAt, + createdAt: archivedAt - 86_400_000, + updatedAt: archivedAt, + }), + ), + ) + const errors = yield* ErrorsService + yield* errors.runTick() + + const [issue] = yield* loadIssues + assert.isDefined(issue) + assert.strictEqual(issue.id, issueId) + assert.isNull(issue.archivedAt) + assert.strictEqual(issue.workflowState, "in_progress") + assert.strictEqual(issue.priority, 1) + assert.strictEqual(issue.occurrenceCount, 43) + assert.strictEqual(issue.fingerprintVersion, FINGERPRINT_VERSION) + }).pipe(Effect.provide(makeTickLayer(database))) + }), + 120_000, + ) +}) diff --git a/packages/backend/src/services/errors/error-tick-persistence.ts b/packages/backend/src/services/errors/error-tick-persistence.ts index 147f06ef28..948be3211b 100644 --- a/packages/backend/src/services/errors/error-tick-persistence.ts +++ b/packages/backend/src/services/errors/error-tick-persistence.ts @@ -663,12 +663,38 @@ export const persistErrorTickWindow = ( resolvedByActorId: null, lastRegressedAt: windowEnd, regressionCount: $.regressionCount.add(1), + // A regression on an issue retention already archived must be seen. + archivedAt: null, updatedAt: windowEnd, })) .where(($) => [$.orgId.eq(input.orgId), $.id.in_(...chunk)]), ), { discard: true }, ) + + // Only retention archives an issue, on the premise that it stopped occurring. + // One that occurs again was retired too early: a fingerprint bump whose new + // hashes are not live yet (a self-hosted warehouse that has not applied the + // view, or a deploy gap), so it comes back with its state intact. A resolved + // one stays archived unless the occurrence is a regression, handled above. + const restoredIds = resolvedRows + .filter( + (entry) => + entry.prior !== undefined && + entry.prior.archivedAt !== null && + entry.prior.workflowState !== "done", + ) + .map((entry) => entry.issueId) + yield* Effect.forEach( + chunks(restoredIds), + (chunk) => + tx.run( + PG.update(ErrorIssues) + .set({ archivedAt: null, updatedAt: windowEnd }) + .where(($) => [$.orgId.eq(input.orgId), $.id.in_(...chunk)]), + ), + { discard: true }, + ) } const observedIssueIds = observed.map((entry) => entry.issueId) diff --git a/packages/backend/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts index 12dd82ff29..dc2c54ad46 100644 --- a/packages/backend/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts @@ -282,17 +282,21 @@ describe.skipIf(!clickhouseE2eEnabled)("error_events attribute fallback (ClickHo it("preserves pre-migration hashes and details for events and status messages", async () => { // Migration 0020 is the last deployed definition of these views before 0030. - // Evaluate its frozen SELECT against identical spans; do not reimplement hashing. - const previousView = migrations - .find((migration) => migration.version === 20) - ?.statements.find( - (statement) => - typeof statement === "string" && - statement.startsWith("CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv "), - ) - assert.isString(previousView) - const previousSelect = previousView.slice(previousView.indexOf(" AS\n") + 4) - const previousRows = await readErrorEvents(`(${previousSelect})`) + // Evaluate both frozen SELECTs against identical spans; do not reimplement + // hashing. (0039 rotates every hash on purpose, so the live view is not 0030.) + const frozenSelect = (version: number): string => { + const view = migrations + .find((migration) => migration.version === version) + ?.statements.find( + (statement) => + typeof statement === "string" && + statement.startsWith("CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv "), + ) + assert.isString(view) + return view.slice(view.indexOf(" AS\n") + 4) + } + const previousRows = await readErrorEvents(`(${frozenSelect(20)})`) + const fallbackRows = await readErrorEvents(`(${frozenSelect(30)})`) for (const spanId of [ "event-wins", "empty-event", @@ -301,12 +305,12 @@ describe.skipIf(!clickhouseE2eEnabled)("error_events attribute fallback (ClickHo "status-and-exception-attrs", "unknown", ]) { - assert.deepStrictEqual(mustGet(rows, spanId), mustGet(previousRows, spanId), spanId) + assert.deepStrictEqual(mustGet(fallbackRows, spanId), mustGet(previousRows, spanId), spanId) } }) it("preserves stored occurrences when the migration is reapplied", async () => { - const migration = migrations.find((entry) => entry.version === 30) + const migration = migrations.find((entry) => entry.version === 39) assert.isDefined(migration) for (const statement of migration.statements) { assert.isString(statement) diff --git a/packages/backend/src/services/warehouse/error-fingerprint-v3.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/error-fingerprint-v3.clickhouse.e2e.test.ts new file mode 100644 index 0000000000..21dacb20fe --- /dev/null +++ b/packages/backend/src/services/warehouse/error-fingerprint-v3.clickhouse.e2e.test.ts @@ -0,0 +1,221 @@ +// SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. +// Fingerprint v3 parity: the hash the error_events MV computes must be +// cityHash64 of exactly the inputs the TS mirror derives, for every v3 rule. + +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { computeFingerprintInputs, FINGERPRINT_HASH_SALT } from "@maple/domain/tinybird/fingerprint" +import { Schema } from "effect" +import { msToDate } from "../../platform/time" +import { + applyRealMigrations, + clickhouseE2eEnabled, + clickhouseExec, + uniqueDatabase, +} from "./clickhouse-e2e-support" + +const database = uniqueDatabase("maple_error_fingerprint_v3_e2e") +const ORG_ID = "org_error_fingerprint_v3" +const SERVICE = "api" + +/** Now-relative: the tables enforce a TTL at insert time. */ +const SEED_TS = msToDate(Date.now() - 60 * 60 * 1000) + .toISOString() + .replace("T", " ") + .slice(0, 19) + +const quote = (value: string): string => `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` + +interface Seed { + readonly spanId: string + readonly type: string + readonly message: string + readonly stacktrace: string +} + +const frames = (...lines: ReadonlyArray) => ["Error: boom", ...lines].join("\n") + +const SEEDS: ReadonlyArray = [ + { + spanId: "namespaced", + type: "@maple/http/errors/UpstreamUnavailable", + message: "upstream 503", + stacktrace: frames( + " at toUpstreamError (/app/src/upstream.ts:40:11)", + " at listTraces (/app/src/traces.ts:12:3)", + ), + }, + { + spanId: "generic-caller-a", + type: "TypeError", + message: "Cannot read properties of undefined (reading 'id')", + stacktrace: frames(" at Array.map ()", " at renderRows (/app/src/table.ts:10:2)"), + }, + { + spanId: "effect-span-frames", + type: "DatabaseError", + message: "connection reset", + stacktrace: frames( + " at query (/app/src/db.ts:5:1)", + " at db.query (definition) (/app/src/db.ts:3:20)", + " at load (/app/src/load.ts:7:2)", + ), + }, + { + spanId: "store-version", + type: "Error", + message: "failed", + stacktrace: frames( + " at fail (/app/node_modules/.bun/effect@4.0.4-beta.1/node_modules/effect/dist/Effect.js:1:1)", + " at Promise.all (index 17)", + ), + }, + { + spanId: "ch-echo", + type: "", + message: "Code: 62. DB::Exception: Syntax error: failed at position 12 (SELECT foo\nFROM bar)", + stacktrace: "", + }, + { + spanId: "ch-envelope", + type: "", + message: "[Error] Unknown identifier `TraceIdd` in scope SELECT TraceIdd FROM traces", + stacktrace: "", + }, + { + spanId: "fs-path", + type: "", + message: "ENOENT: no such file or directory, open '/tmp/maple-a81f/wal.log' (=/var/run/x.sock)", + stacktrace: "", + }, + { + spanId: "route-path", + type: "", + message: "Transport error (POST /api/data/export) for /Users/riordan/.maple/store", + stacktrace: "", + }, + { + spanId: "ids", + type: "", + message: "no customer for org_2NnAbCdEfGhIjKlM, run 01HZY3K4W7Q9N2B6C8D0E5F1GA, cus_QwErTyUiOpAsDf", + stacktrace: "", + }, + { + spanId: "quoted-table", + type: "", + message: "Table 'checkout_events' doesn't exist", + stacktrace: "", + }, + { + spanId: "json-uuid", + type: "", + message: JSON.stringify({ error: "not_found", id: "550e8400-e29b-41d4-a716-446655440000", n: 42 }), + stacktrace: "", + }, + // v3.1 + { + spanId: "namespaced-empty-message", + type: "@maple/discord/RateLimited", + message: "", + stacktrace: frames(" at fail (/app/src/fail.ts:4:1)", " at send (/app/src/send.ts:9:2)"), + }, + { + spanId: "bundle-frames", + type: "Error", + message: "failed", + stacktrace: frames( + ' at [2]["params"]', + " at a (file:///bundle/worker.js:1:2)", + " at b (https://x.dev/assets/index-AbCd1234-EfGh5678.js:1:2)", + " at c (https://x.dev/_astro/Layout.AbCd1234.js:1:2)", + ), + }, + { + spanId: "ansi", + type: "", + message: "\x1b[31mconnection refused\x1b[0m", + stacktrace: "", + }, + { + spanId: "ch-while-processing", + type: "", + message: + "DB::Exception: Illegal type String of argument of function plus: While processing plus(a, 1)", + stacktrace: "", + }, + { + spanId: "path-variants", + type: "", + message: + "copy /data.quarantine-x1/store /var/b, ~/.maple/x, read /datasets/x, retry in ~5 min, key top_operations", + stacktrace: "", + }, + ...[401, 404, 500, 502, 520, 524].map((status) => ({ + spanId: `http-${status}`, + type: "", + message: `upstream returned ${status}, HTTP ${status} after 3 tries`, + stacktrace: "", + })), +] + +const seed = async (): Promise => { + const rows = SEEDS.map( + (row) => + `(${quote(ORG_ID)}, ${quote(SEED_TS)}, ${quote(`trace-${row.spanId}`)}, ${quote(row.spanId)}, '', 'GET /', 'Server', ${quote(SERVICE)}, 1000000, 'Error', ${quote(row.message)}, map(), map(), [toDateTime64(${quote(SEED_TS)}, 9)], ['exception'], [map('exception.type', ${quote(row.type)}, 'exception.message', ${quote(row.message)}, 'exception.stacktrace', ${quote(row.stacktrace)})])`, + ).join(",\n") + await clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, StatusMessage, SpanAttributes, ResourceAttributes, EventsTimestamp, EventsName, EventsAttributes) + VALUES\n${rows}`, + database, + ) +} + +const HashRow = Schema.Struct({ SpanId: Schema.String, TopFrame: Schema.String, Hash: Schema.String }) +type HashRow = typeof HashRow.Type +const decodeHashRow = Schema.decodeUnknownSync(Schema.fromJsonString(HashRow)) + +const readRows = async (sql: string): Promise> => { + const body = await clickhouseExec(`${sql} FORMAT JSONEachRow`, database) + const rows = body + .split("\n") + .filter((line) => line.trim().length > 0) + .map(decodeHashRow) + return new Map(rows.map((row) => [row.SpanId, row])) +} + +/** cityHash64 over the TS mirror's inputs, evaluated by ClickHouse itself. */ +const mirroredSql = SEEDS.map((row) => { + const inputs = computeFingerprintInputs({ + exceptionType: row.type, + exceptionStacktrace: row.stacktrace, + statusMessage: row.message, + }) + return `SELECT ${quote(row.spanId)} AS SpanId, ${quote(inputs.topFrame)} AS TopFrame, toString(cityHash64(${quote(ORG_ID)}, ${quote(SERVICE)}, ${quote(row.type)}, ${quote(inputs.fpFrames)}, ${quote(inputs.msgSignature)}, ${quote(FINGERPRINT_HASH_SALT)})) AS Hash` +}).join("\nUNION ALL\n") + +describe.skipIf(!clickhouseE2eEnabled)("error fingerprint v3 (ClickHouse e2e)", () => { + let materialized: Map + let mirrored: Map + + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE IF NOT EXISTS ${database}`) + await applyRealMigrations(database) + await seed() + materialized = await readRows( + `SELECT SpanId, TopFrame, toString(FingerprintHash) AS Hash FROM error_events WHERE OrgId = ${quote(ORG_ID)}`, + ) + mirrored = await readRows(mirroredSql) + }, 180_000) + + afterAll(async () => { + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }) + + for (const { spanId } of SEEDS) { + it(`hashes ${spanId} exactly as the TS mirror does`, () => { + const actual = materialized.get(spanId) + assert.isDefined(actual, `${spanId} was not materialized`) + assert.deepStrictEqual(actual, mirrored.get(spanId)) + }) + } +}) diff --git a/packages/domain/src/clickhouse/ddl-emitter.test.ts b/packages/domain/src/clickhouse/ddl-emitter.test.ts index 38b3c2d895..04480e1412 100644 --- a/packages/domain/src/clickhouse/ddl-emitter.test.ts +++ b/packages/domain/src/clickhouse/ddl-emitter.test.ts @@ -1,5 +1,6 @@ import { Effect } from "effect" import { describe, expect, it } from "vitest" +import { FINGERPRINT_HASH_SALT } from "../tinybird/fingerprint" import { buildTinybirdProjectManifest } from "../tinybird/project-manifest" import { emitCreateMaterializedView, @@ -90,7 +91,9 @@ describe("ClickHouse DDL emitter", () => { expect(ddl).toMatch(/^CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS/) expect(ddl).toContain("FROM traces") expect(ddl).toContain("WHERE StatusCode = 'Error'") - expect(ddl).toContain("cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig)") + expect(ddl).toContain( + `cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, '${FINGERPRINT_HASH_SALT}')`, + ) }) it("emits a JSONPath spec mapping each ingested column to its $.path", async () => { diff --git a/packages/domain/src/clickhouse/migrations/0039_error_fingerprint_v3.ts b/packages/domain/src/clickhouse/migrations/0039_error_fingerprint_v3.ts new file mode 100644 index 0000000000..4aa8ae152c --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0039_error_fingerprint_v3.ts @@ -0,0 +1,26 @@ +/** + * Migration 0039: error fingerprint v3. + * + * A namespaced exception type keeps only its throw site, Effect span frames + * are dropped, frames lose package versions and `(index N)`, and the message + * signature cuts ClickHouse query echoes and redacts filesystem paths, prefixed + * ids and ULIDs. The hash also takes the version salt (`'fp3'`), so every + * FingerprintHash rotates and the FINGERPRINT_VERSION sweep archives only issues + * that can never recur. + * + * Forward-only: stored rows keep their v2 hashes until the TTL ages them out. + * `requiredForIngest: false`: the gateway never writes these targets. The + * CREATE statements are the verbatim DDL the schema emitter produced at v39; + * never re-derive them from a later snapshot. + */ +export const migration_0039_error_fingerprint_v3 = { + version: 39, + description: "Recreate the error_events MVs with fingerprint v3 grouping rules and a version-salted hash", + requiredForIngest: false, + statements: [ + "DROP VIEW IF EXISTS error_events_mv", + "DROP VIEW IF EXISTS error_events_by_time_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\['),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\['),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 2dd7dd76cb..b4034182c6 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -37,6 +37,7 @@ import { migration_0035_ai_trace_index_gateway_stamps } from "./0035_ai_trace_in import { migration_0036_alert_checks_skip_reason } from "./0036_alert_checks_skip_reason" import { migration_0037_service_overview_spans_span_kind } from "./0037_service_overview_spans_span_kind" import { migration_0038_external_edges_legacy_db_system } from "./0038_external_edges_legacy_db_system" +import { migration_0039_error_fingerprint_v3 } from "./0039_error_fingerprint_v3" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -106,6 +107,7 @@ export const migrations: ReadonlyArray = [ migration_0036_alert_checks_skip_reason, migration_0037_service_overview_spans_span_kind, migration_0038_external_edges_legacy_db_system, + migration_0039_error_fingerprint_v3, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 9133604e77..05531a680b 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "6946cd8703d07c0be3ef010ccbde01194cbfd3ac8e324dd6f36da99132a6fbba" as const +export const projectRevision = "a9e17246b1cf0b92e8af1cb6a16606c9de70698233d82bb295231da6e22da702" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_crawler_requests (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n ServiceName LowCardinality(String),\n Crawler LowCardinality(String),\n Host LowCardinality(String),\n Path String,\n HttpStatus UInt16\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -48,8 +48,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS ai_crawler_requests_mv TO ai_crawler_requests AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n ServiceName,\n arrayElement(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'Meta-ExternalAgent', 'Meta-WebIndexer', 'Meta-ExternalFetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'])) AS Crawler,\n lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')) AS Host,\n leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path,\n toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus\n FROM traces\n WHERE SpanKind = 'Server'\n AND multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai']) > 0\n AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS\nSELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanAttributes['maple_ai.session.id'] AS SessionId,\n SpanAttributes['maple_ai.vendor.id'] AS VendorId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n SpanAttributes['maple_ai.model'] AS Model,\n SpanAttributes['maple_ai.agent.name'] AS AgentName,\n SpanAttributes['maple_ai.tool.name'] AS ToolName,\n SpanId,\n ParentSpanId,\n Duration,\n toUInt8(SpanAttributes['maple_ai.error'] = '1') AS IsError,\n toUInt8(SpanAttributes['maple_ai.llm_call'] = '1') AS IsLlmCall,\n toUInt8(SpanAttributes['maple_ai.tool_call'] = '1') AS IsToolCall,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS Tokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cost']) AS Cost,\n SpanAttributes['maple_ai.response.id'] AS ResponseId,\n SpanAttributes['maple_ai.vendor.version'] AS VendorVersion,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) AS InputTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) AS CacheReadTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) AS CacheWriteTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) AS OutputTokens,\n toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS ReasoningTokens,\n SpanAttributes['error.type'] AS ErrorType,\n leftUTF8(StatusMessage, 400) AS StatusMessage,\n SpanAttributes['maple_ai.tool.description'] AS ToolDescription,\n SpanAttributes['maple_ai.tool.error_result'] AS FailedToolCallResult,\n if(SpanAttributes['maple_ai.error'] = '1', cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(leftUTF8(coalesce(nullIf(SpanAttributes['maple_ai.tool.error_result'], ''), leftUTF8(StatusMessage, 400)), 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#')), 0) AS ErrorFingerprint\n FROM traces\n WHERE SpanAttributes['maple_ai.vendor.id'] != ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\['),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\['),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", "CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS\nSELECT\n OrgId,\n VisitorId,\n UserId,\n StartTime AS FirstSeen\n FROM session_replays\n WHERE VisitorId != '' AND UserId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 50a625d5e3..65ce1ac2da 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "6946cd8703d07c0be3ef010ccbde01194cbfd3ac8e324dd6f36da99132a6fbba" as const +export const projectRevision = "a9e17246b1cf0b92e8af1cb6a16606c9de70698233d82bb295231da6e22da702" as const export const datasources = [ { @@ -230,12 +230,12 @@ export const pipes = [ { name: "error_events_by_time_mv", content: - "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time\nDEPLOYMENT_METHOD alter", + "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\['),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time\nDEPLOYMENT_METHOD alter", }, { name: "error_events_mv", content: - "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event (falling back to exception.* / error.* span attributes) and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events\nDEPLOYMENT_METHOD alter", + "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event (falling back to exception.* / error.* span attributes) and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n -- Only fill the old Unknown Error bucket. Event values (including\n -- empty fields) and spans with StatusMessage keep every hash input.\n _ei = 0 AND StatusMessage = '' AS _useAttrs,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.type'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '')\n ) AS _exType,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.message'],\n if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage)\n ) AS _exMsg,\n if(\n _ei > 0, EventsAttributes[_ei]['exception.stacktrace'],\n if(_useAttrs, SpanAttributes['exception.stacktrace'], '')\n ) AS _exStack,\n if(_useAttrs, _exMsg, StatusMessage) AS _msgText,\n -- JSON detection for the message signature below.\n isValidJSON(_msgText) AS _isJson,\n _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F-]{6,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(_msgText)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 400), '\\\\x1b\\\\[[0-9;]*m', ''), '^((\\\\[Error\\\\]|[\\\\s\\\\S]*?DB::Exception:)[\\\\s\\\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\\\s\\\\S]*', '\\\\1'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])~/[^ \\'\"`:,)]*', '\\\\1PATH'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '(^|[ \\\\t\\\\r\\\\n\\'\"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+|home/[^/ \\\\t\\\\r\\\\n\\'\"`:,)]+)([/.][^ \\\\t\\\\r\\\\n\\'\"`:,)]*)?([ \\\\t\\\\r\\\\n\\'\"`:,)]|$)', '\\\\1PATH\\\\4'), '\\\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\\\b', 'ID'), '\\\\b[0-9A-HJKMNP-TV-Z]{26}\\\\b', 'ID'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)([1-4][0-9][0-9]|500)\\\\b', '\\\\1\\\\2§\\\\3'), '\\\\b(HTTP|[Ss]tatus|returned|code|failed|failed with)(:? ?)5(0[1-9]|[1-9][0-9])\\\\b', '\\\\1\\\\2§5xx'), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\\\1'),\n 1, 120\n )\n ) AS _msgSig,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+')\n AND position(line, '(definition)') = 0\n AND NOT match(line, '^[ \\\\t]*at \\\\['),\n splitByChar('\\n', _exStack)\n ),\n -- A namespaced type plus its message names one failure: keep only the\n -- throw site. With no message the frames are all that is left; keep 3.\n 1, if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3)\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@[0-9]+\\\\.[0-9]+\\\\.[0-9]+[^/ )]*', '@#'), '\\\\(index [0-9]+\\\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(-[A-Za-z0-9_-]{8})+\\\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\\\.css', '.css'), '(/_astro/[^/ ).]+)\\\\.[A-Za-z0-9_-]{8}\\\\.js', '\\\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'),\n JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'),\n JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'),\n JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'),\n JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'),\n JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'),\n JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n _msgText = '', 'Unknown Error',\n position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0,\n if(\n extract(_msgText, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(_msgText, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel,\n left(_msgText, multiIf(\n position(_msgText, ': ') > 3, toInt64(position(_msgText, ': ')) - 1,\n position(_msgText, ' (') > 3, toInt64(position(_msgText, ' (')) - 1,\n position(_msgText, '\\n') > 3, toInt64(position(_msgText, '\\n')) - 1,\n least(toInt64(length(_msgText)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp3') AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all hold: 4xx, no exception event, no exception.type\n -- attribute, and no error.type beyond the status code itself (HTTP\n -- semconv sets error.type to the bare status on a non-2xx response,\n -- which carries no exception). 5xx and anything carrying a real\n -- exception still count, and SpanKind is deliberately not consulted —\n -- these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND SpanAttributes['exception.type'] = ''\n AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus))\n )\n\nTYPE MATERIALIZED\nDATASOURCE error_events\nDEPLOYMENT_METHOD alter", }, { name: "error_fingerprints_minutely_mv", diff --git a/packages/domain/src/tinybird/fingerprint.test.ts b/packages/domain/src/tinybird/fingerprint.test.ts index 82c8a6aee6..0e67f30116 100644 --- a/packages/domain/src/tinybird/fingerprint.test.ts +++ b/packages/domain/src/tinybird/fingerprint.test.ts @@ -2,13 +2,17 @@ import { describe, expect, it } from "vitest" import { chPattern, computeFingerprintInputs, + EFFECT_SPAN_FRAME_MARKER, + ERROR_MSG_REDACTIONS, + FINGERPRINT_HASH_SALT, FRAME_LINE_PATTERN, FRAME_REDACTIONS, JSON_VALUE_REDACTIONS, MAX_FINGERPRINT_FRAMES, MSG_SCAN_CHARS, MSG_SIGNATURE_CHARS, - MSG_TEXT_REDACTIONS, + NAMESPACED_TYPE_MAX_FRAMES, + SCHEMA_PATH_LINE_PATTERN, } from "./fingerprint" import { ERROR_EVENTS_MV_SQL } from "./materializations" @@ -464,7 +468,7 @@ describe("error fingerprint normalization", () => { expect(a.msgSignature).toContain("/api/query-engine/overview") }) - it("collapses per-user home directories but keeps the rest of the path", () => { + it("collapses a home-directory path to PATH, so two users' stores are one bug", () => { const a = computeFingerprintInputs({ exceptionType: "IngestRejected", exceptionStacktrace: "", @@ -478,7 +482,7 @@ describe("error fingerprint normalization", () => { }) expect(a.msgSignature).toBe(b.msgSignature) - expect(a.msgSignature).toContain("/.maple/data/store") + expect(a.msgSignature).toBe("Cannot open file PATH: No space left on device") }) it("redacts an email address out of the signature", () => { @@ -582,15 +586,24 @@ describe("SQL parity", () => { it("renders the frame matcher and every redaction from the shared constants", () => { expect(sql).toContain(`match(line, ${chPattern(FRAME_LINE_PATTERN)})`) + expect(sql).toContain(`position(line, ${chPattern(EFFECT_SPAN_FRAME_MARKER)}) = 0`) + expect(sql).toContain(`NOT match(line, ${chPattern(SCHEMA_PATH_LINE_PATTERN)})`) + // The frame limit reads _msgSig, so the signature must be defined first. + expect(sql.indexOf("AS _msgSig")).toBeLessThan(sql.indexOf("AS _rawFrames")) for (const [pattern, replacement] of [ ...FRAME_REDACTIONS, ...JSON_VALUE_REDACTIONS, - ...MSG_TEXT_REDACTIONS, + ...ERROR_MSG_REDACTIONS, ]) { - expect(sql).toContain(`${chPattern(pattern)}, ${chPattern(replacement)}`) + const chReplacement = replacement.replace(/\$([1-9])/g, "\\$1") + expect(sql).toContain(`${chPattern(pattern)}, ${chPattern(chReplacement)}`) } }) + it("hashes the version salt, so a FINGERPRINT_VERSION bump rotates every issue", () => { + expect(sql).toContain(`_msgSig, ${chPattern(FINGERPRINT_HASH_SALT)}) AS FingerprintHash`) + }) + it("truncates by character in both implementations, not by byte", () => { // ClickHouse `substring` counts bytes while JS `slice` counts UTF-16 units, // so a non-ASCII message would truncate at a different point on each side. @@ -600,28 +613,34 @@ describe("SQL parity", () => { }) it("keeps the frame limit in step", () => { - expect(sql).toContain(`1, ${MAX_FINGERPRINT_FRAMES}\n ) AS _rawFrames`) + expect(sql).toContain( + `1, if(position(_exType, '/') > 0 AND _msgSig != '', ${NAMESPACED_TYPE_MAX_FRAMES}, ${MAX_FINGERPRINT_FRAMES})\n ) AS _rawFrames`, + ) }) it("emits patterns ClickHouse's RE2 can compile", () => { // RE2 rejects backreferences and lookaround; JS accepts both, so a pattern // written here can typecheck, pass every test above, and still fail at // deploy time against the warehouse. - for (const [pattern] of [...FRAME_REDACTIONS, ...JSON_VALUE_REDACTIONS, ...MSG_TEXT_REDACTIONS]) { + for (const [pattern] of [...FRAME_REDACTIONS, ...JSON_VALUE_REDACTIONS, ...ERROR_MSG_REDACTIONS]) { expect(pattern).not.toMatch(/\(\?[=!<]/) expect(pattern).not.toMatch(/\\[1-9]/) } expect(FRAME_LINE_PATTERN).not.toMatch(/\(\?[=!<]/) }) - it("uses replacements neither engine reinterprets", () => { - for (const [, replacement] of [ + it("uses no replacement syntax beyond a group reference both engines share", () => { + // `$1` (JS) and `\1` (ClickHouse) are the one form rendered across; `$&`, + // `$` or a literal backslash would mean different things on each side. + for (const [pattern, replacement] of [ ...FRAME_REDACTIONS, ...JSON_VALUE_REDACTIONS, - ...MSG_TEXT_REDACTIONS, + ...ERROR_MSG_REDACTIONS, ]) { - expect(replacement).not.toContain("$") - expect(replacement).not.toContain("\\") + const groups = [...replacement.matchAll(/\$([1-9])/g)].map((m) => Number(m[1])) + expect(replacement.replace(/\$[1-9]/g, "")).not.toMatch(/[$\\]/) + const capturing = new RegExp(`${pattern}|`).exec("")?.length ?? 1 + for (const group of groups) expect(group).toBeLessThan(capturing) } }) }) @@ -729,3 +748,224 @@ describe("Apple crash frames", () => { } }) }) + +describe("fingerprint v3", () => { + const key = (exceptionType: string, exceptionStacktrace: string, statusMessage: string) => { + const inputs = computeFingerprintInputs({ exceptionType, exceptionStacktrace, statusMessage }) + return `${inputs.fpFrames}\u0000${inputs.msgSignature}` + } + const stack = (...frames: ReadonlyArray) => ["Error: boom", ...frames].join("\n") + + describe("frames", () => { + it("keeps only the throw site for a namespaced type, so its callers are one issue", () => { + const throwSite = " at toUpstreamError (/app/src/upstream.ts:40:11)" + const a = stack( + throwSite, + " at listTraces (/app/src/traces.ts:12:3)", + " at handler (/app/src/http.ts:9:1)", + ) + const b = stack( + throwSite, + " at serviceMap (/app/src/map.ts:88:7)", + " at handler (/app/src/http.ts:9:1)", + ) + const type = "@maple/http/errors/UpstreamUnavailable" + + expect(key(type, a, "upstream 503")).toBe(key(type, b, "upstream 503")) + expect( + computeFingerprintInputs({ + exceptionType: type, + exceptionStacktrace: a, + statusMessage: "upstream 503", + }).fpFrames, + ).toBe(" at toUpstreamError (/app/src/upstream.ts)") + }) + + it("keeps all 3 frames for a namespaced type with no message, since frames are all that is left", () => { + const throwSite = " at fail (/app/src/fail.ts:4:1)" + const a = stack(throwSite, " at listTraces (/app/src/traces.ts:12:3)") + const b = stack(throwSite, " at serviceMap (/app/src/map.ts:88:7)") + const type = "@maple/discord/RateLimited" + + expect(key(type, a, "")).not.toBe(key(type, b, "")) + }) + + it("ignores a schema error's key path as a frame", () => { + const withPath = stack(' at [2]["params"]', " at decode (/app/src/schema.ts:9:4)") + + expect(key("ParseError", withPath, "")).toBe( + key("ParseError", stack(" at decode (/app/src/schema.ts:9:4)"), ""), + ) + }) + + it("strips file:///bundle/, repeated bundle hashes and Astro asset hashes", () => { + const top = (line: string) => + computeFingerprintInputs({ + exceptionType: "Error", + exceptionStacktrace: stack(line), + statusMessage: "", + }).topFrame + + expect(top(" at a (file:///bundle/worker.js:1:2)")).toBe(" at a (worker.js)") + expect(top(" at a (https://x.dev/assets/index-AbCd1234-EfGh5678.js:1:2)")).toBe( + " at a (/assets/index.js)", + ) + expect(top(" at a (https://x.dev/_astro/Layout.AbCd1234.js:1:2)")).toBe( + " at a (/_astro/Layout.js)", + ) + }) + + it("keeps generic TypeErrors thrown at different callers apart", () => { + const a = stack(" at Array.map ()", " at renderRows (/app/src/table.ts:10:2)") + const b = stack(" at Array.map ()", " at renderChart (/app/src/chart.ts:31:9)") + const message = "Cannot read properties of undefined (reading 'id')" + + expect(key("TypeError", a, message)).not.toBe(key("TypeError", b, message)) + }) + + it("drops Effect span frames, so span depth does not split one call path", () => { + const plain = stack(" at query (/app/src/db.ts:5:1)", " at load (/app/src/load.ts:7:2)") + const withSpans = stack( + " at query (/app/src/db.ts:5:1)", + " at db.query (definition) (/app/src/db.ts:3:20)", + " at load (/app/src/load.ts:7:2)", + ) + + expect(key("DatabaseError", withSpans, "")).toBe(key("DatabaseError", plain, "")) + }) + + it("redacts the package version in a store path", () => { + const at = (version: string) => + stack( + ` at fail (/app/node_modules/.bun/effect@${version}/node_modules/effect/dist/Effect.js:1:1)`, + ) + + expect(key("Error", at("4.0.3"), "")).toBe(key("Error", at("4.0.4-beta.1"), "")) + expect( + computeFingerprintInputs({ + exceptionType: "Error", + exceptionStacktrace: at("4.0.3"), + statusMessage: "", + }).topFrame, + ).toContain("effect@#/") + }) + + it("redacts the index in `(index N)`", () => { + const at = (index: number) => stack(` at Promise.all (index ${index})`) + + expect(key("Error", at(3), "")).toBe(key("Error", at(17), "")) + }) + }) + + describe("message signature", () => { + const signature = (statusMessage: string) => + computeFingerprintInputs({ exceptionType: "", exceptionStacktrace: "", statusMessage }) + .msgSignature + + it("cuts only the query a ClickHouse error echoes, keeping the identifier that names the bug", () => { + const echo = (query: string) => + signature( + `Code: 47. DB::Exception: Unknown expression identifier \`ServiceVersion\` in scope ${query}`, + ) + + expect(echo("SELECT ServiceVersion FROM traces")).toBe( + "Code: #. DB::Exception: Unknown expression identifier `ServiceVersion`", + ) + expect(echo("SELECT * FROM (SELECT ServiceVersion FROM logs)")).toBe( + echo("SELECT ServiceVersion FROM traces"), + ) + expect( + signature( + "DB::Exception: Illegal type String of argument of function plus: While processing plus(a, 1)", + ), + ).toBe("DB::Exception: Illegal type String of argument of function plus:") + }) + + it("keeps different missing identifiers apart", () => { + expect(signature("[Error] Unknown identifier `TraceIdd` in scope SELECT 1")).not.toBe( + signature("[Error] Unknown identifier `Severity` in scope SELECT 1"), + ) + }) + + it("cuts the echoed query from the `[Error]` envelope", () => { + expect( + signature("[Error] Unknown identifier `TraceIdd` in scope SELECT TraceIdd FROM traces"), + ).toBe("[Error] Unknown identifier `TraceIdd`") + }) + + it("strips terminal colour codes", () => { + expect(signature("\x1b[31mconnection refused\x1b[0m")).toBe("connection refused") + }) + + it("ends a filesystem root at a separator, so `/data.x` is a path and `/datasets` is not", () => { + expect(signature("cannot open /data.quarantine-x1/store")).toBe( + signature("cannot open /data/store"), + ) + expect(signature("read /datasets/x failed")).toBe("read /datasets/x failed") + }) + + it("replaces `~/` paths and adjacent paths, but not a `~` before a number", () => { + expect(signature("open ~/.maple/data failed")).toBe("open PATH failed") + expect(signature("copy /tmp/a /var/b")).toBe("copy PATH PATH") + expect(signature("retry in ~5 min")).toBe("retry in ~# min") + }) + + it("keeps a snake_case name with no digit or capital", () => { + expect(signature("unknown key top_operations")).toBe("unknown key top_operations") + }) + + it("keeps a 4xx and 500 exact and folds every other 5xx into one class", () => { + const returned = (status: number) => signature(`upstream returned ${status}`) + + expect(new Set([returned(401), returned(404), returned(500)]).size).toBe(3) + expect(returned(502)).toBe("upstream returned 5xx") + expect(returned(520)).toBe(returned(502)) + expect(returned(524)).toBe(returned(502)) + expect(signature("status: 429 too many")).toBe("status: 429 too many") + expect(signature("HTTP 503 from gateway")).toBe("HTTP 5xx from gateway") + }) + + it("replaces filesystem-rooted paths with PATH", () => { + expect(signature("ENOENT: no such file or directory, open '/tmp/maple-a81f/wal.log'")).toBe( + "ENOENT: no such file or directory, open 'PATH'", + ) + expect(signature("socket /var/run/maple/abc.sock closed")).toBe( + signature("socket /var/run/maple/xyz.sock closed"), + ) + }) + + it("keeps route paths, including ones that contain a filesystem root name", () => { + expect(signature("Transport error (POST /api/data/export)")).toBe( + "Transport error (POST /api/data/export)", + ) + expect(signature("Transport error (POST /api/query-engine/service-overview)")).not.toBe( + signature("Transport error (POST /api/query-engine/service-map)"), + ) + }) + + it("keeps different quoted table names apart", () => { + expect(signature("Table 'checkout_events' doesn't exist")).not.toBe( + signature("Table 'planetscale_events' doesn't exist"), + ) + }) + + it("replaces prefixed ids and ULIDs with ID", () => { + expect(signature("no billing customer for org_2NnAbCdEfGhIjKlM")).toBe( + "no billing customer for ID", + ) + expect(signature("subscription cus_QwErTyUiOpAsDf lapsed")).toBe( + signature("subscription cus_ZxCvBnMlKjHgFd lapsed"), + ) + expect(signature("run 01HZY3K4W7Q9N2B6C8D0E5F1GA failed")).toBe("run ID failed") + }) + + it("redacts UUIDs, dashes included, inside a JSON value", () => { + const json = (id: string) => JSON.stringify({ error: "not_found", id }) + + expect(signature(json("550e8400-e29b-41d4-a716-446655440000"))).toBe( + signature(json("9b2f1c3e-0d4a-4f6b-8c7e-1a2b3c4d5e6f")), + ) + expect(signature(json("550e8400-e29b-41d4-a716-446655440000"))).toBe('error="not_found"|id="#"') + }) + }) +}) diff --git a/packages/domain/src/tinybird/fingerprint.ts b/packages/domain/src/tinybird/fingerprint.ts index 772881dcda..68d36effd6 100644 --- a/packages/domain/src/tinybird/fingerprint.ts +++ b/packages/domain/src/tinybird/fingerprint.ts @@ -52,12 +52,21 @@ export interface FingerprintInputs { * constants, so a change here reaches both or neither. * * Everything must stay inside the RE2 subset ClickHouse supports: no - * backreferences, no lookaround. Replacements must contain no `$` (JS) and no - * backslash (ClickHouse), since both treat those as capture-group syntax. + * backreferences in a pattern, no lookaround. A replacement may name a capture + * group as `$1`..`$9`, written the JS way; `chRedactChain` renders it as `\1`. + * No other `$` and no backslash may appear in a replacement. */ /** Frames considered for the hash, from the top of the stack down. */ export const MAX_FINGERPRINT_FRAMES = 3 +/** + * Frames kept for a namespaced type (`@maple/http/errors/Foo`, `effect/ai/AiError`). + * Such a type already names one failure, so frames 2-3 are only its callers: one + * upstream 503 became one issue per query that happened to hit it. + */ +export const NAMESPACED_TYPE_MAX_FRAMES = 1 +/** Marks an Effect span frame, which varies the stack depth of one call path. */ +export const EFFECT_SPAN_FRAME_MARKER = "(definition)" /** How much of StatusMessage is scanned before redaction. */ export const MSG_SCAN_CHARS = 400 /** How much of the redacted result reaches the hash. */ @@ -105,14 +114,19 @@ export type Redactions = ReadonlyArray `'${value.replace(/\\/g, "\\ export const chRedactChain = (expr: string, redactions: Redactions): string => redactions.reduce( (inner, [pattern, replacement]) => - `replaceRegexpAll(${inner}, ${chPattern(pattern)}, ${chPattern(replacement)})`, + `replaceRegexpAll(${inner}, ${chPattern(pattern)}, ${chPattern(replacement.replace(/\$([1-9])/g, "\\$1"))})`, expr, ) /** - * Apply a redaction list in TypeScript. The replacement is passed as a function - * so JS never reinterprets `$&` and friends — ClickHouse would not. + * Apply a redaction list in TypeScript. `$1`..`$9` expand to capture groups, as + * `\1`..`\9` do in ClickHouse. */ export const applyRedactions = (value: string, redactions: Redactions): string => redactions.reduce( - (acc, [pattern, replacement]) => acc.replace(new RegExp(pattern, "g"), () => replacement), + (acc, [pattern, replacement]) => acc.replace(new RegExp(pattern, "g"), replacement), value, ) const FRAME_LINE_RE = new RegExp(FRAME_LINE_PATTERN) +/** ` at [2]["params"]`: a schema error's key path, not a stack frame. */ +export const SCHEMA_PATH_LINE_PATTERN = "^[ \\t]*at \\[" +const SCHEMA_PATH_LINE_RE = new RegExp(SCHEMA_PATH_LINE_PATTERN) // Display-only candidate keys for the human label (NOT used by the fingerprint). const LABEL_KEYS = ["title", "message", "error", "_tag", "reason", "name"] as const @@ -278,7 +339,7 @@ function normalizeFrame(line: string): string { function messageSignature(statusMessage: string): string { const obj = tryParseJsonObject(statusMessage) if (obj !== undefined) return jsonSignature(obj) - return applyRedactions(statusMessage.slice(0, MSG_SCAN_CHARS), MSG_TEXT_REDACTIONS).slice( + return applyRedactions(statusMessage.slice(0, MSG_SCAN_CHARS), ERROR_MSG_REDACTIONS).slice( 0, MSG_SIGNATURE_CHARS, ) @@ -290,17 +351,27 @@ export function computeFingerprintInputs(args: { readonly exceptionStacktrace: string readonly statusMessage: string }): FingerprintInputs { + const msgSignature = messageSignature(args.statusMessage) + // A namespaced type plus its message names the failure, so the throw site is enough. + // With no message the frames are the only identity left; keep all three. + const maxFrames = + args.exceptionType.includes("/") && msgSignature !== "" + ? NAMESPACED_TYPE_MAX_FRAMES + : MAX_FINGERPRINT_FRAMES const rawFrames = args.exceptionStacktrace .split("\n") - .filter((line) => FRAME_LINE_RE.test(line)) - .slice(0, MAX_FINGERPRINT_FRAMES) + .filter( + (line) => + FRAME_LINE_RE.test(line) && + !line.includes(EFFECT_SPAN_FRAME_MARKER) && + !SCHEMA_PATH_LINE_RE.test(line), + ) + .slice(0, maxFrames) const topFrames = rawFrames.map(normalizeFrame) const topFrame = topFrames[0] ?? "" const fpFrames = topFrames.join("\n") - const msgSignature = messageSignature(args.statusMessage) - const label = args.exceptionType !== "" ? args.exceptionType : statusLabel(args.statusMessage) return { topFrame, fpFrames, msgSignature, label } @@ -316,37 +387,31 @@ export function computeFingerprintInputs(args: { * stale rows retire deterministically, instead of sitting in `triage` until the * 14-day auto-resolve window catches up. * + * That archive assumes a bump rotates EVERY hash, so since v3 the version itself + * is hashed, as `FINGERPRINT_HASH_SALT`. Where an old hash is still produced + * anyway (a self-hosted warehouse that has not applied the new view, or a deploy + * gap), the tick restores an archived open issue on its next occurrence and + * restamps its version. (The Apple frame alternative and the attribute fallback + * predate both and skipped a bump for that reason.) + * * Deliberately NOT a ClickHouse column. Old and new hashes cannot collide, so - * the warehouse needs no discriminator — only the Postgres row does, and keeping - * it there means a future bump is a constant change plus a sweep, with no - * datasource migration or materialized-view backfill. + * the warehouse needs no discriminator — only the Postgres row does. * * Changing the view does NOT rewrite history: rows already in `error_events` - * keep their v1 hashes for the rest of their 90-day TTL, so a v2 issue's detail - * view, occurrence sparkline and sample traces all start empty and fill from - * cutover forward. That is expected, not a regression — there is no backfill - * short of replaying the raw traces. - * - * v1 → v2 (this change): frame lines are matched by shape rather than by - * "contains a colon-digit", and the message signature is always folded in and - * additionally strips quoted values and query strings. + * keep their old hashes for the rest of their 90-day TTL, so a new issue's + * detail view, occurrence sparkline and sample traces all start empty and fill + * from cutover forward. That is expected, not a regression. * - * **Adding the Apple frame alternative deliberately did NOT bump this**, even - * though it rotates hashes for iOS crashes that are still occurring, which is - * what the rule above otherwise calls for. The retirement this version drives is - * version-keyed, not hash-keyed: `ErrorsService` archives every `kind: "error"` - * issue whose `fingerprintVersion` is below this constant, on the premise stated - * in `error_issues.fingerprintVersion` that a row on an older version can never - * receive another occurrence. That premise holds only when a bump rotates - * *every* hash. The Apple change rotates iOS hashes alone, so a bump would - * archive every Node, Python, Go and browser issue in every org — and nothing - * un-archives them, because the tick's upsert conflicts on - * `(orgId, fingerprintHash)` and never clears `archivedAt`. + * v1 → v2: frame lines are matched by shape rather than by "contains a + * colon-digit", and the message signature is always folded in and additionally + * strips quoted values and query strings. * - * The cost of not bumping is small and bounded: the collapsed iOS issues stop - * receiving occurrences the moment their hash changes, and retire through the - * ordinary resolved window instead of on sight. Teaching the sweep to retire by - * hash rather than by version is what would let a partial-rotation change bump - * this safely. + * v2 → v3: a namespaced type keeps only its throw site, Effect span frames are + * dropped, frames lose package versions and `(index N)`, the message signature + * cuts ClickHouse query echoes and redacts filesystem paths, prefixed ids and + * ULIDs, JSON values lose UUIDs, and the version salt enters the hash. */ -export const FINGERPRINT_VERSION = 2 +export const FINGERPRINT_VERSION = 3 + +/** Hashed into `FingerprintHash` so a version bump rotates every issue. */ +export const FINGERPRINT_HASH_SALT = `fp${FINGERPRINT_VERSION}` diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 633b274124..b2c297b663 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -2,13 +2,17 @@ import { defineMaterializedView, node } from "@maple-dev/effect-orm/tinybird" import { chPattern, chRedactChain, + EFFECT_SPAN_FRAME_MARKER, + ERROR_MSG_REDACTIONS, + FINGERPRINT_HASH_SALT, FRAME_LINE_PATTERN, FRAME_REDACTIONS, JSON_VALUE_REDACTIONS, MAX_FINGERPRINT_FRAMES, MSG_SCAN_CHARS, MSG_SIGNATURE_CHARS, - MSG_TEXT_REDACTIONS, + NAMESPACED_TYPE_MAX_FRAMES, + SCHEMA_PATH_LINE_PATTERN, } from "./fingerprint" import { serviceUsage, @@ -732,8 +736,8 @@ export const servicePlatformsHourlyMv = defineMaterializedView("service_platform * are absent, reads `exception.*` then `error.*` span attributes. Computes a * cityHash64 FingerprintHash used to group occurrences into Issues. * - * Fingerprint inputs: (OrgId, ServiceName, ExceptionType, top-3 normalized frames, - * message signature). + * Fingerprint inputs: (OrgId, ServiceName, ExceptionType, top-3 normalized frames + * (1 for a namespaced type), message signature, FINGERPRINT_HASH_SALT). * - Stack lines are filtered by frame SHAPE — one alternative per runtime's frame * syntax — not by "contains `:NUMBER`". The old rule accepted any colon-digit * line, so Drizzle's `params:` line (actual row values) and `Type: message` @@ -791,37 +795,6 @@ const errorEventsSelectSql = ` if(_useAttrs, SpanAttributes['exception.stacktrace'], '') ) AS _exStack, if(_useAttrs, _exMsg, StatusMessage) AS _msgText, - -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old - -- rule accepted any line with a colon-digit, which let non-frame lines - -- in: Drizzle's \`params: \` line, and the \`Type: message\` - -- header (\`Code: 62\`, \`position 1628\`, embedded timestamps). Row values - -- and message text then entered the hash and split one bug into - -- thousands of issues — 23,035 fingerprints for six real - -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError - -- ones. - -- - -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, - -- as is every redaction below. They used to be hand-copied here, which - -- let the reference implementation the tests exercise drift away from - -- the SQL that actually runs, silently. - arraySlice( - arrayFilter( - line -> match(line, ${chPattern(FRAME_LINE_PATTERN)}), - splitByChar('\\n', _exStack) - ), - 1, ${MAX_FINGERPRINT_FRAMES} - ) AS _rawFrames, - -- Redact every volatile token a frame line can carry: the URL origin - -- (so preview hosts share one fingerprint), Vite's 8-char bundle - -- content hash (so a deploy does not re-split every triaged browser and - -- Worker issue), then line numbers, hex pointers and long id runs. See - -- FRAME_REDACTIONS for the order and the reasoning. - arrayMap( - line -> ${chRedactChain("line", FRAME_REDACTIONS)}, - _rawFrames - ) AS _topFrames, - if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, - arrayStringConcat(_topFrames, '\\n') AS _fpFrames, -- JSON detection for the message signature below. isValidJSON(_msgText) AS _isJson, _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, @@ -847,15 +820,50 @@ const errorEventsSelectSql = ` -- restores that discrimination, and it cannot reinflate cardinality the -- way a raw prefix would because everything variable is redacted first: -- emails, URL origins, home directories, query strings, quoted values, - -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, -- what is deliberately kept, and the one residual it cannot reach. multiIf( _isJsonObj, _jsonSig, substringUTF8( - ${chRedactChain(`substringUTF8(_msgText, 1, ${MSG_SCAN_CHARS})`, MSG_TEXT_REDACTIONS)}, + ${chRedactChain(`substringUTF8(_msgText, 1, ${MSG_SCAN_CHARS})`, ERROR_MSG_REDACTIONS)}, 1, ${MSG_SIGNATURE_CHARS} ) ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's \`params: \` line, and the \`Type: message\` + -- header (\`Code: 62\`, \`position 1628\`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, ${chPattern(FRAME_LINE_PATTERN)}) + AND position(line, ${chPattern(EFFECT_SPAN_FRAME_MARKER)}) = 0 + AND NOT match(line, ${chPattern(SCHEMA_PATH_LINE_PATTERN)}), + splitByChar('\\n', _exStack) + ), + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + 1, if(position(_exType, '/') > 0 AND _msgSig != '', ${NAMESPACED_TYPE_MAX_FRAMES}, ${MAX_FINGERPRINT_FRAMES}) + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> ${chRedactChain("line", FRAME_REDACTIONS)}, + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\\n') AS _fpFrames, -- Display-only, best-effort human label (decoupled from the fingerprint: -- many labels may map to one hash). The broad key list here is a DISPLAY -- heuristic only; the fingerprint above makes no key-name assumption. @@ -906,7 +914,7 @@ const errorEventsSelectSql = ` _exMsg AS ExceptionMessage, _exStack AS ExceptionStacktrace, _topFrame AS TopFrame, - cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, ${chPattern(FINGERPRINT_HASH_SALT)}) AS FingerprintHash, StatusMessage, Duration, _errorLabel AS ErrorLabel, diff --git a/packages/query-engine/src/execution/executor.ts b/packages/query-engine/src/execution/executor.ts index 8ddc610072..e4700e7a5b 100644 --- a/packages/query-engine/src/execution/executor.ts +++ b/packages/query-engine/src/execution/executor.ts @@ -585,7 +585,7 @@ WHERE name = 'enable_full_text_index'`, // this client timeout into the retry loop. Effect.fail( new WarehouseQueryError({ - message: `Warehouse query exceeded ${attemptTimeoutMs}ms client timeout`, + message: `Warehouse query ${pipe} exceeded ${attemptTimeoutMs}ms client timeout`, pipeName: pipe, }), ), diff --git a/scripts/errors-replay/candidate.ts b/scripts/errors-replay/candidate.ts index e13cf2177e..fa5f058ca8 100644 --- a/scripts/errors-replay/candidate.ts +++ b/scripts/errors-replay/candidate.ts @@ -1,32 +1,13 @@ /** - * A PROPOSED fingerprint (v3), replayed against real samples before it ships. + * A PROPOSED fingerprint, replayed against real samples before it ships. * * Nothing in production reads this. It exists so a grouping change can be - * measured (`bun run errors:replay`) instead of argued. When a proposal wins, - * port it to `packages/domain/src/tinybird/fingerprint.ts` and the MV SQL, and - * bump FINGERPRINT_VERSION. - * - * Each rule below targets a split seen in prod (2026-10-13 sample): - * 1. Caller frames: one upstream 503 became one issue per query that hit it, - * because frames 2-3 are the callers. A namespaced error type already names - * the failure, so only the throw site is kept for those. - * 2. Effect span frames (`(definition)`) vary the stack depth for one call path. - * 3. Filesystem paths outside the home dir (`/tmp/...`, `/srv/...`) and the - * random suffixes inside them. Route paths (`/api/...`) are still kept. - * 4. Prefixed and non-hex ids (`org_2Nn...`, `cus_...`, ULIDs) and UUIDs inside JSON. + * measured (`bun run errors:replay`) instead of argued. It starts as a copy of + * production (v3, `packages/domain/src/tinybird/fingerprint.ts`): edit + * `candidateInputs` to propose a change. When a proposal wins, port it to + * fingerprint.ts and the MV SQL, bump FINGERPRINT_VERSION, and reset this file. */ -import { Option, Schema } from "effect" -import { - applyRedactions, - computeFingerprintInputs, - FRAME_LINE_PATTERN, - FRAME_REDACTIONS, - MAX_FINGERPRINT_FRAMES, - MSG_SCAN_CHARS, - MSG_SIGNATURE_CHARS, - MSG_TEXT_REDACTIONS, - type Redactions, -} from "../../packages/domain/src/tinybird/fingerprint" +import { computeFingerprintInputs } from "../../packages/domain/src/tinybird/fingerprint" export interface ReplayEvent { readonly serviceName: string @@ -35,81 +16,23 @@ export interface ReplayEvent { readonly exceptionStacktrace: string } -const FRAME_LINE_RE = new RegExp(FRAME_LINE_PATTERN) - -/** `@maple/http/errors/Foo`, `effect/ai/AiError`: a type that names one failure. */ -const isNamespacedType = (type: string) => type.includes("/") - -const isEffectSpanFrame = (line: string) => line.includes("(definition)") - -const CANDIDATE_FRAME_REDACTIONS: Redactions = [ - // pnpm/bun store paths carry the package version; a dependency bump is not a new bug. - ["@[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*", "@#"], - ["\\(index [0-9]+\\)", "(index #)"], - ...FRAME_REDACTIONS, -] - -const FS_ROOTS = "tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run" - -// Replacements may use `$1`; ClickHouse's replaceRegexpAll spells it `\1`. No -// lookbehind: these must port to RE2 for the MV. -const CANDIDATE_MSG_REDACTIONS: Redactions = [ - // ClickHouse errors echo the failing query; the error class is the bug, the - // query text (an agent's own SQL, mostly) is a value. - ["(DB::Exception: [A-Za-z ]+?)[:.(][\\s\\S]*", "$1"], - ["^(\\[Error\\] [A-Za-z ]+?)[`'\"#:.(][\\s\\S]*", "$1"], - // Filesystem paths, rooted where a route never is, and only at the start of a - // token so `/api/data/x` is not read as `/data`. Runs before the home-dir pass. - [`(^|[\\s'"(\`=])(~|/(${FS_ROOTS}|Users/[^/ ]+|home/[^/ ]+))(/[^ '"\`:,)]*)?`, "$1PATH"], - // Prefixed ids: Clerk `org_`/`user_`, Stripe `cus_`/`sub_`, and their like. - ["\\b[a-z]{2,8}_[A-Za-z0-9]{10,}\\b", "ID"], - // ULIDs: 26 Crockford base32 characters. - ["\\b[0-9A-HJKMNP-TV-Z]{26}\\b", "ID"], - ...MSG_TEXT_REDACTIONS, -] - -const CANDIDATE_JSON_VALUE_REDACTIONS: Redactions = [["[0-9a-fA-F-]{6,}|[0-9]+", "#"]] - -function candidateFrames(event: ReplayEvent): string { - const keep = isNamespacedType(event.exceptionType) ? 1 : MAX_FINGERPRINT_FRAMES - return event.exceptionStacktrace - .split("\n") - .filter((line) => FRAME_LINE_RE.test(line) && !isEffectSpanFrame(line)) - .slice(0, keep) - .map((line) => applyRedactions(line, CANDIDATE_FRAME_REDACTIONS)) - .join("\n") +interface KeyInputs { + readonly fpFrames: string + readonly msgSignature: string } -function candidateMessage(message: string): string { - const json = parseJsonObject(message) - if (json !== undefined) { - return Object.keys(json) - .map((key) => `${key}=${applyRedactions(JSON.stringify(json[key]), CANDIDATE_JSON_VALUE_REDACTIONS)}`) - .sort() - .join("|") - } - return applyWithGroups(message.slice(0, MSG_SCAN_CHARS), CANDIDATE_MSG_REDACTIONS).slice(0, MSG_SIGNATURE_CHARS) -} - -const applyWithGroups = (value: string, redactions: Redactions): string => - redactions.reduce((acc, [pattern, replacement]) => acc.replace(new RegExp(pattern, "g"), replacement), value) - -const decodeJsonObject = Schema.decodeUnknownOption( - Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), -) +const key = (event: ReplayEvent, inputs: KeyInputs): string => + [event.serviceName, event.exceptionType, inputs.fpFrames, inputs.msgSignature].join("\u0000") -const parseJsonObject = (text: string): Record | undefined => - Option.getOrUndefined(decodeJsonObject(text)) +/** The proposal. No proposal is open, so this is production. */ +const candidateInputs = (event: ReplayEvent): KeyInputs => computeFingerprintInputs(event) /** The grouping key production computes today (the TS mirror of the MV). */ export function currentKey(event: ReplayEvent): string { - const inputs = computeFingerprintInputs(event) - return [event.serviceName, event.exceptionType, inputs.fpFrames, inputs.msgSignature].join("\u0000") + return key(event, computeFingerprintInputs(event)) } /** The proposed grouping key. */ export function candidateKey(event: ReplayEvent): string { - return [event.serviceName, event.exceptionType, candidateFrames(event), candidateMessage(event.statusMessage)].join( - "\u0000", - ) + return key(event, candidateInputs(event)) }