diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7928ece..140c957 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,3 +19,4 @@ jobs: - run: npm run test -w @autter/runtime-browser - run: npm run test -w @autter/otlp-ingester - run: npm run size -w @autter/runtime-browser + - run: python3 -m unittest discover -s adapters/python -v diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c0cd1b..4b54180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [Unreleased] + +### Security + +- Scrub secrets and PII from exception messages, stack traces and span status messages, not only custom attributes; re-scrub every span at export so third-party instrumentations, HTTP URLs (`?token=`) and `recordException` calls are covered; scrub `LlmCallHandle.setAttributes`, `instrumentLlmClient` attributes and provider errors, and module-level captures made before `initAutterServer` (`runtime-node`). +- New value patterns everywhere: Basic auth, `Authorization:`/`Cookie:`/`Set-Cookie:` header text, connection strings with empty usernames or `@` in the password, `sk-proj-`/`sk-ant-`, Stripe, GitHub fine-grained, GitLab, npm, SendGrid and Google API keys, `password=`/`?token=`/`?api_key=` assignments, Luhn-valid card numbers. New sensitive keys: any `…authorization` header key, `session`/`sessionid`/`sid`, `pwd`, `dsn`, bounded `ssn`. Custom value patterns now apply globally (`runtime-node`, `runtime-browser`, `otlp-ingester`). +- The browser relay scrubs message, stack, name and nested context (`redact` option) (`runtime-node`). +- Browser SDK scrubs messages, stacks and nested context; new `redact: { keys, values } | false` option and `scrubText` export (`runtime-browser`). +- Ingester scrubs occurrence message/stack/route, span names, browser context (deep) and LLM-call attributes before storage and the sink webhook; extra patterns via `AUTTER_REDACT_VALUE_PATTERNS` / `AUTTER_REDACT_KEY_PATTERNS` (`otlp-ingester`). +- Python adapter: new stdlib-only `adapters/python/redact.py` (`redact_text`, `redact_attributes`, `configure`); the caught-exception sampler scrubs tracebacks with it. +- Shared parity vectors in `test-vectors/redaction.json` run against every implementation. + +### Behavior notes + +- Error messages that contained secrets now fingerprint on their scrubbed form, so such issues may regroup once (they previously split per secret value). + ## [1.3.1] - 2026-09-08 ### Fixes diff --git a/adapters/python/caught_exceptions.py b/adapters/python/caught_exceptions.py index 86de6df..1b2b481 100644 --- a/adapters/python/caught_exceptions.py +++ b/adapters/python/caught_exceptions.py @@ -4,18 +4,26 @@ ones. It has substantial overhead and must be explicitly enabled. Pass an ``emit(name, stack)`` callback that records a standard OTel exception event with ``autter.handled=true`` and ``autter.sampled=true`` on an always-on span. + +The formatted traceback includes ``str(exc)``, so it is scrubbed for secrets +and PII (connection strings, tokens, API keys, emails, …) with +:func:`redact.redact_text` before ``emit`` sees it. """ from __future__ import annotations import random -import re import sys import threading import time import traceback from collections.abc import Callable +try: # imported as part of a package + from .redact import redact_text +except ImportError: # copied next to the app as a plain module + from redact import redact_text + def install_caught_sampler( emit: Callable[[str, str], None], *, sample_rate: float = 0.01, max_per_minute: int = 10 @@ -43,8 +51,7 @@ def hook(frame, event, arg): return hook seen.add(identity) sent += 1 - stack = "".join(traceback.format_exception(exc_type, exc_value, tb, limit=8))[:8000] - stack = re.sub(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", "[redacted]", stack, flags=re.I) + stack = redact_text("".join(traceback.format_exception(exc_type, exc_value, tb, limit=8))[:8000]) try: emit(exc_type.__name__, stack) except Exception: diff --git a/adapters/python/redact.py b/adapters/python/redact.py new file mode 100644 index 0000000..2b45cb9 --- /dev/null +++ b/adapters/python/redact.py @@ -0,0 +1,198 @@ +"""Secret/PII scrubbing for Python services reporting to Autter Runtime. + +Python exceptions routinely carry secrets in their *message* (``str(exc)``) +and therefore in every formatted traceback: ``psycopg2.OperationalError: +connection to postgres://admin:hunter2@db failed``, ``401 Incorrect API key +provided: sk-...``. Scrub before anything leaves the process: + + from redact import redact_text, redact_attributes + + span.add_event("exception", { + "exception.type": type(exc).__name__, + "exception.message": redact_text(str(exc)), + "exception.stacktrace": redact_text("".join(traceback.format_exception(exc))), + }) + span.set_attributes(redact_attributes(custom_context)) + +Same patterns as ``redactText``/``redactAttributes`` in @autter/runtime-node; +parity is enforced by the shared ``test-vectors/redaction.json``. Secure by +default; extend with :func:`configure` (your own value/key patterns). The +Autter ingester scrubs again server-side as a second line of defence. + +Stdlib only. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +MASK = "[redacted]" + +_PRIVATE_KEY_BLOCK = re.compile( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)" +) +_PREFIXED_SECRET = re.compile( + r"\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}" + r"|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}" + r"|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}" + r"|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}" + r"|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})" +) +_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}") +_AUTH_SCHEME = re.compile( + r"\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}" + r"|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})", + re.I, +) +_HEADER_VALUE = re.compile( + r"((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)[\"']?\s*[:=]\s*[\"']?)[^\"'\r\n]+", + re.I | re.M, +) +_SECRET_ASSIGNMENT = re.compile( + r"(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key" + r"|credential|signature|session[_-]?id|sessionid|ssn)" + r"([\"']?\s*[:=]\s*)" + r"(\"[^\"\r\n]*\"|'[^'\r\n]*'|[^\s\"'&,;)}\]\[<>]+)", + re.I, +) +_URL_CREDENTIALS = re.compile( + r"\b([a-z][a-z0-9+.-]{0,31}://)(?:[^\s/:@\"'<>]*:[^\s/\"'<>]*|[^\s/:@\"'<>]{16,})@", re.I +) +_CARD = re.compile( + r"\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b" +) +_EMAIL = re.compile(r"[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}", re.I) + +_SENSITIVE_KEYS = [ + re.compile(p) + for p in ( + r"e-?mail", + r"pass(word|wd|phrase)|^pass$|(^|[._-])pwd$", + r"token", + r"secret", + r"credential", + r"(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key", + r"authori[sz]ation|^auth(-|_|$)|bearer", + r"cookie", + r"(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid", + r"phone|msisdn", + r"(^|[^a-z])ssn($|[^a-z])|social[-_ ]?security", + r"cvv|cvc|card([-_. ]?(number|num|no))?$", + r"credit[-_.]?card", + r"connection[-_.]?string|(^|[._-])dsn$", + r"recovery[-_.]?code|\botp\b|magic[-_.]?link", + ) +] +# Numeric GenAI usage counters keep their value even though "token" matches. +_USAGE_KEY = re.compile(r"(^|\.)(input|output|total|prompt|completion)_?tokens$|token_count$", re.I) + +_extra_values: list[re.Pattern[str]] = [] +_extra_keys: list[re.Pattern[str]] = [] +_enabled = True + +_MAX_DEPTH = 8 +_MAX_ITEMS = 1000 + + +def configure( + *, + value_patterns: Iterable[str | re.Pattern[str]] = (), + key_patterns: Iterable[str | re.Pattern[str]] = (), + enabled: bool = True, +) -> None: + """Add your own patterns on top of the built-ins (replaces earlier extras). + + ``value_patterns`` are masked wherever they occur inside strings; + ``key_patterns`` mask whole values whose key matches (case-insensitive). + ``enabled=False`` turns scrubbing off (not recommended). + """ + global _extra_values, _extra_keys, _enabled + _extra_values = [p if isinstance(p, re.Pattern) else re.compile(p, re.I) for p in value_patterns] + _extra_keys = [p if isinstance(p, re.Pattern) else re.compile(p, re.I) for p in key_patterns] + _enabled = enabled + + +def _luhn_valid(candidate: str) -> bool: + digits = [int(c) for c in candidate if c.isdigit()] + if not 13 <= len(digits) <= 19: + return False + total = 0 + for i, d in enumerate(reversed(digits)): + if i % 2 == 1: + d *= 2 + if d > 9: + d -= 9 + total += d + return total % 10 == 0 + + +def _mask_assignment(match: re.Match[str]) -> str: + key, sep, value = match.group(1), match.group(2), match.group(3) + if value[:1] in ("'", '"'): + return f"{key}{sep}{value[0]}{MASK}{value[0]}" + return f"{key}{sep}{MASK}" + + +def redact_text(value: str) -> str: + """Scrub secrets/PII embedded in one string; the rest survives.""" + if not _enabled or not isinstance(value, str) or not value: + return value + out = _PRIVATE_KEY_BLOCK.sub(MASK, value) + out = _PREFIXED_SECRET.sub(MASK, out) + out = _JWT.sub(MASK, out) + out = _AUTH_SCHEME.sub(MASK, out) + out = _HEADER_VALUE.sub(lambda m: m.group(1) + MASK, out) + out = _SECRET_ASSIGNMENT.sub(_mask_assignment, out) + out = _URL_CREDENTIALS.sub(lambda m: m.group(1) + MASK + "@", out) + out = _CARD.sub(lambda m: MASK if _luhn_valid(m.group(0)) else m.group(0), out) + out = _EMAIL.sub(MASK, out) + for pattern in _extra_values: + out = pattern.sub(MASK, out) + return out + + +def is_sensitive_key(key: str) -> bool: + lowered = key.lower() + return any(p.search(lowered) for p in _SENSITIVE_KEYS) or any(p.search(key) for p in _extra_keys) + + +def _redact_value(key: str, value: Any, depth: int) -> Any: + if isinstance(value, bool) or value is None: + return value + if isinstance(value, (int, float)): + if _USAGE_KEY.search(key) and value >= 0: + return value + return MASK if is_sensitive_key(key) else value + if is_sensitive_key(key): + return MASK + if isinstance(value, str): + return redact_text(value) + if depth >= _MAX_DEPTH: + return MASK + if isinstance(value, Mapping): + return _redact_mapping(value, depth + 1) + if isinstance(value, (list, tuple)): + return [_redact_value("", item, depth + 1) for item in list(value)[:_MAX_ITEMS]] + return redact_text(str(value)) + + +def _redact_mapping(mapping: Mapping[Any, Any], depth: int) -> dict[str, Any]: + out: dict[str, Any] = {} + for i, (key, value) in enumerate(mapping.items()): + if i >= _MAX_ITEMS: + out["__redaction_truncated__"] = MASK + break + out[str(key)] = _redact_value(str(key), value, depth) + return out + + +def redact_attributes(attributes: Mapping[str, Any] | None) -> dict[str, Any]: + """Copy of ``attributes`` with sensitive keys masked (any depth) and + secrets scrubbed inside string values. Never mutates the input.""" + if not attributes: + return {} + if not _enabled: + return dict(attributes) + return _redact_mapping(attributes, 0) diff --git a/adapters/python/test_redact.py b/adapters/python/test_redact.py new file mode 100644 index 0000000..2f42694 --- /dev/null +++ b/adapters/python/test_redact.py @@ -0,0 +1,111 @@ +"""Tests for the Python adapter's redaction (stdlib unittest, no deps). + +Run: python3 -m unittest discover -s adapters/python -v +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import redact # noqa: E402 +from caught_exceptions import install_caught_sampler # noqa: E402 +from redact import MASK, configure, redact_attributes, redact_text # noqa: E402 + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "test-vectors", "redaction.json") +with open(VECTORS_PATH, encoding="utf-8") as fh: + VECTORS = json.load(fh) + + +def _join(value): + return "".join(value) if isinstance(value, list) else value + + +class SharedVectors(unittest.TestCase): + def test_text_vectors(self): + for vector in VECTORS["text"]: + if "python" in vector.get("skip", []): + continue + with self.subTest(vector["name"]): + self.assertEqual(redact_text(_join(vector["input"])), _join(vector["expect"])) + + def test_sensitive_keys_masked_at_any_depth(self): + for key in VECTORS["keys"]["sensitive"]: + with self.subTest(key): + out = redact_attributes({"outer": {"items": [{key: "raw"}]}}) + self.assertEqual(out["outer"]["items"][0][key], MASK) + + def test_safe_keys_survive(self): + for key in VECTORS["keys"]["safe"]: + with self.subTest(key): + self.assertEqual(redact_attributes({"outer": {key: "kept"}})["outer"][key], "kept") + + +class Attributes(unittest.TestCase): + def test_nested_values_and_usage_counters(self): + out = redact_attributes( + { + "db": {"dsn": "postgres://u:p@h/db", "args": ["redis://:pw@cache:6379"]}, + "gen_ai.usage.input_tokens": 12, + "max_tokens": 100, + "note": "contact jane@example.com", + } + ) + self.assertEqual(out["db"]["dsn"], MASK) + self.assertEqual(out["db"]["args"], [f"redis://{MASK}@cache:6379"]) + self.assertEqual(out["gen_ai.usage.input_tokens"], 12) + self.assertEqual(out["max_tokens"], MASK) + self.assertEqual(out["note"], f"contact {MASK}") + + def test_does_not_mutate_input(self): + original = {"password": "x", "nested": {"token": "y"}} + redact_attributes(original) + self.assertEqual(original, {"password": "x", "nested": {"token": "y"}}) + + def test_custom_patterns(self): + configure(value_patterns=[r"CUST-\d{4}"], key_patterns=[r"^tenant_ref$"]) + try: + self.assertEqual(redact_text("CUST-1234 / CUST-9999"), f"{MASK} / {MASK}") + self.assertEqual(redact_attributes({"tenant_ref": "t"}), {"tenant_ref": MASK}) + finally: + configure() + + def test_opt_out(self): + configure(enabled=False) + try: + self.assertEqual(redact_text("password=x"), "password=x") + finally: + configure() + self.assertTrue(redact._enabled) + + +class CaughtExceptionSampler(unittest.TestCase): + def test_traceback_message_is_scrubbed_before_emit(self): + emitted: list[tuple[str, str]] = [] + + def connect(): + raise ConnectionError("could not connect to postgres://admin:hunter2@db.internal:5432/app") + + stop = install_caught_sampler(lambda name, stack: emitted.append((name, stack)), sample_rate=1.0) + try: + try: + connect() + except ConnectionError: + pass + finally: + stop() + + self.assertTrue(emitted, "sampler emitted nothing") + name, stack = emitted[0] + self.assertEqual(name, "ConnectionError") + self.assertNotIn("hunter2", stack) + self.assertIn(f"postgres://{MASK}@db.internal:5432/app", stack) + self.assertIn("in connect", stack) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c038fb7..1ad0ed1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -222,11 +222,20 @@ counters, not an analytics event store). Forbidden at the schema level (rejected/stripped): full URLs with query strings, cookies, DOM content, form values, request headers/bodies, emails. -Server-side custom attributes are guarded at the source instead: the Node -SDK masks email/token/credential-shaped values and sensitive-keyed -attributes before export (`redactAttributes`, on by default), so OTLP spans -never carry a stray `user.email` even though the OTLP schema itself accepts -free-form attributes. +Server-side telemetry is guarded at the source instead: the Node SDK masks +secret/PII-shaped values (tokens, keys, credentials in connection strings +and URLs, emails, card numbers) and sensitive-keyed attributes in +attributes, exception messages, stack traces and status messages, and +re-scrubs every span at export (`redactAttributes`, on by default). The +browser SDK, the relay and the Python adapter apply the same patterns. + +Because old SDKs and plain OTel senders scrub nothing, the ingester scrubs +again before storage and before the sink webhook (which feeds downstream +LLM-assisted fixing): occurrence message/stack/route, span names, all +attribute bags, and LLM-call attributes (`src/redact.ts`; extend with +`AUTTER_REDACT_VALUE_PATTERNS` / `AUTTER_REDACT_KEY_PATTERNS`, JSON arrays +of regex sources). Messages containing secrets therefore fingerprint on +their scrubbed form. ## Sink webhook (v1) diff --git a/docs/CONTINUOUS-DETECTION.md b/docs/CONTINUOUS-DETECTION.md index 5410629..823e5eb 100644 --- a/docs/CONTINUOUS-DETECTION.md +++ b/docs/CONTINUOUS-DETECTION.md @@ -67,6 +67,14 @@ OTel `exception` event with `autter.handled=true` and overhead, applies to the current and newly created threads, and should be enabled only for targeted diagnosis. Other languages can send the same event. +The sampler scrubs each traceback (which includes `str(exc)`) for secrets +and PII before calling `emit`, using `adapters/python/redact.py` (stdlib +only; copy it next to `caught_exceptions.py`). Use the same module for your +own `record_exception` calls — `redact_text(str(exc))`, +`redact_text(traceback_text)`, `redact_attributes(context)` — and +`configure(value_patterns=[...], key_patterns=[...])` for custom patterns. +The ingester scrubs again server-side. + ```python from opentelemetry import trace from caught_exceptions import install_caught_sampler diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 1f3a360..a993628 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -66,6 +66,8 @@ The validator webhook may return the same extra fields: | `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) | | `OCCURRENCE_TTL_DAYS` / `SPAN_TTL_DAYS` / `METRICS_TTL_DAYS` | `14` / `7` / `90` | ClickHouse TTLs (applied at table creation) | | `LLM_CALL_TTL_DAYS` | `90` | Retention for `runtime_llm_calls` rows | +| `AUTTER_REDACT_VALUE_PATTERNS` | — | JSON array of extra regex sources masked in stored/forwarded text and attribute values (built-in secret/PII patterns always apply) | +| `AUTTER_REDACT_KEY_PATTERNS` | — | JSON array of extra regex sources for attribute keys whose values are masked wholesale | ## LLM / GenAI calls diff --git a/packages/otlp-ingester/src/config.ts b/packages/otlp-ingester/src/config.ts index 45d1d5c..e64679f 100644 --- a/packages/otlp-ingester/src/config.ts +++ b/packages/otlp-ingester/src/config.ts @@ -38,6 +38,15 @@ export interface IngesterConfig { spanTtlDays: number; metricsTtlDays: number; llmCallTtlDays: number; + /** + * Extra regex sources scrubbed from stored/forwarded free text and + * attribute values, and extra regex sources for sensitive attribute + * keys — on top of the built-in secret/PII patterns (see redact.ts). + * Env: AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS (JSON + * arrays of strings). + */ + redactValuePatterns?: string[]; + redactKeyPatterns?: string[]; } function intEnv(name: string, fallback: number): number { @@ -47,6 +56,19 @@ function intEnv(name: string, fallback: number): number { return Number.isFinite(value) && value > 0 ? value : fallback; } +/** JSON array of regex sources; invalid input fails startup loudly rather + * than silently running without the operator's extra patterns. */ +function patternListEnv(name: string): string[] { + const raw = process.env[name]; + if (!raw) return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed) || !parsed.every((p) => typeof p === "string")) { + throw new Error(`${name} must be a JSON array of regex source strings`); + } + for (const source of parsed) new RegExp(source as string); + return parsed as string[]; +} + function parseIngestKeys(raw: string | undefined): StaticIngestKey[] { if (!raw) return []; try { @@ -91,6 +113,8 @@ export function loadConfig(): IngesterConfig { // LLM calls keep the metrics horizon, not the span one — cost trends // need months, and per-call volume is small next to HTTP spans. llmCallTtlDays: intEnv("LLM_CALL_TTL_DAYS", 90), + redactValuePatterns: patternListEnv("AUTTER_REDACT_VALUE_PATTERNS"), + redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"), }; if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) { throw new Error( diff --git a/packages/otlp-ingester/src/context.ts b/packages/otlp-ingester/src/context.ts index ff5e30d..df80d9e 100644 --- a/packages/otlp-ingester/src/context.ts +++ b/packages/otlp-ingester/src/context.ts @@ -1,22 +1,15 @@ +import { isSensitiveKey, scrubText } from "./redact.js"; + /** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */ export function sanitizeRuntimeContext( input: unknown, ): Record { let budget = 512; const seen = new WeakSet(); + // Shared secret/PII patterns (redact.ts), then drop URL query strings + // and fragments outright — whitelist-style privacy for free text. const scrub = (value: string) => - value - .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted]") - .replace( - /\b(?:bearer\s+[A-Za-z0-9._~+\/=~-]{10,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|(?:AKIA|ASIA)[A-Z0-9]{16})\b/gi, - "[redacted]", - ) - .replace( - /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g, - "[redacted]", - ) - .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi, "$1[redacted]@") - .replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); + scrubText(value).replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); const visit = (value: unknown, key: string, depth: number): unknown => { if (--budget < 0 || depth > 6) return "[truncated]"; if (/__proto__|constructor|prototype/i.test(key)) return undefined; @@ -25,13 +18,7 @@ export function sanitizeRuntimeContext( typeof value === "number" && Number.isFinite(value) && value >= 0; - if ( - !usage && - /password|passwd|secret|token|credential|authorization|cookie|email|phone|ssn|card[._-]?number|connection[._-]?string|api[._-]?key|private[._-]?key|request[._-]?body|response[._-]?body|headers|url[._-]?query/i.test( - key, - ) - ) - return "[redacted]"; + if (!usage && key && isSensitiveKey(key)) return "[redacted]"; if (typeof value === "string") { // Serialized custom context crosses the same privacy boundary as nested values. if (/^\s*[\[{]/.test(value)) { diff --git a/packages/otlp-ingester/src/llm.ts b/packages/otlp-ingester/src/llm.ts index a2ad074..a797c3a 100644 --- a/packages/otlp-ingester/src/llm.ts +++ b/packages/otlp-ingester/src/llm.ts @@ -1,4 +1,5 @@ import { estimateLlmCostUsd } from "./llm-pricing.js"; +import { isSensitiveKey, scrubText } from "./redact.js"; import type { RuntimeLlmCall } from "./types.js"; /** @@ -160,7 +161,7 @@ export function extractLlmCall( 200, ) : "", - userId: ( + userId: scrubText( strAttr( attrs, "autter.user_id", @@ -168,21 +169,27 @@ export function extractLlmCall( "ai.telemetry.metadata.user_id", "enduser.id", "user.id", - ) ?? "" + ) ?? "", ).slice(0, 200), - sessionId: ( + sessionId: scrubText( strAttr( attrs, "autter.session_id", "ai.telemetry.metadata.sessionId", "session.id", - ) ?? "" + ) ?? "", ).slice(0, 200), attributes: llmAttributeSubset(attrs), startedAt: facts.startedAt, }; } +/** Numeric token/usage counters are kept even though their keys contain + * "token" — cost tracking depends on them. */ +function isUsageCount(key: string, value: string): boolean { + return /tokens?$|token_?count$|usage/i.test(key) && /^\d+(\.\d+)?$/.test(value); +} + /** * Drill-down context stored as JSON alongside the typed columns: only the * LLM-relevant attribute families, size-capped so a prompt accidentally @@ -201,7 +208,14 @@ function llmAttributeSubset( ) { continue; } - const trimmed = value.length > 500 ? `${value.slice(0, 500)}…` : value; + // Prompts/completions (ai.prompt, gen_ai.prompt, …) land here: scrub + // before the size cap so a secret straddling it can't half-survive. + const safe = isUsageCount(key, value) + ? value + : isSensitiveKey(key) + ? "[redacted]" + : scrubText(value); + const trimmed = safe.length > 500 ? `${safe.slice(0, 500)}…` : safe; size += key.length + trimmed.length; if (size > 4000) break; subset[key] = trimmed; diff --git a/packages/otlp-ingester/src/normalize-browser.ts b/packages/otlp-ingester/src/normalize-browser.ts index 847d58e..a6e5d92 100644 --- a/packages/otlp-ingester/src/normalize-browser.ts +++ b/packages/otlp-ingester/src/normalize-browser.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { sanitizeRuntimeContext } from "./context.js"; import { normalizeRoute } from "./fingerprint.js"; +import { scrubText } from "./redact.js"; import { asSeverity, type RuntimeMetricPoint, @@ -65,26 +67,19 @@ const TYPE_TO_ERROR_TYPE: Record = { }; // Content-level gate for the free-form `context` bag. The schema whitelist -// above is structural; this masks obvious PII/secrets inside whatever a -// (possibly outdated) SDK still sends: values under sensitive-looking keys -// and email-shaped strings — mirroring redactAttributes() in -// @autter/runtime-node and redactContext() in @autter/runtime-browser. -const SENSITIVE_KEY_RE = - /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +// above is structural; this masks secrets/PII inside whatever a (possibly +// outdated) SDK still sends — sensitive keys and secret-shaped values at any +// depth, via the same sanitiser as OTLP attributes. Message, stack, and route +// are scrubbed centrally before fingerprinting (server.ts fingerprintAll). +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; const REDACTED = "[redacted]"; function scrubContext(context: Record): Record { - const out: Record = {}; - for (const [key, value] of Object.entries(context)) { - if (value === undefined || value === null) continue; - out[key] = SENSITIVE_KEY_RE.test(key) - ? REDACTED - : typeof value === "string" - ? value.replace(EMAIL_VALUE_RE, REDACTED) - : value; - } - return out; + return sanitizeRuntimeContext( + Object.fromEntries( + Object.entries(context).filter(([, value]) => value !== undefined && value !== null), + ), + ); } /** Default severity per event type when the SDK doesn't say. */ @@ -113,6 +108,7 @@ export function normalizeBrowserPayload( occurredAt: Date, counts: { requestCount?: number; errorCount?: number; sessionCount?: number }, ): void { + route = scrubText(route); const bucketAt = new Date( Math.floor(occurredAt.getTime() / 60_000) * 60_000, ); @@ -156,7 +152,7 @@ export function normalizeBrowserPayload( const name = event.name === "browser.longtask" ? "browser.longtask" : `browser.resource:${normalizeRoute((event.name ?? "").replace(/^browser\.resource:/, "").split("?")[0]!.replace(EMAIL_VALUE_RE, REDACTED))}`; bumpRollup(name, occurredAt, { requestCount: 1 }); - const point = rollups.get(`${name} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`); + const point = rollups.get(`${scrubText(name)} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`); if (point) point.durationSumMs += event.durationMs ?? 0; continue; } diff --git a/packages/otlp-ingester/src/normalize-otlp.ts b/packages/otlp-ingester/src/normalize-otlp.ts index e939916..09cb307 100644 --- a/packages/otlp-ingester/src/normalize-otlp.ts +++ b/packages/otlp-ingester/src/normalize-otlp.ts @@ -1,4 +1,5 @@ import { decodeOtlpAttributes, sanitizeRuntimeContext, type OtlpValue } from "./context.js"; +import { scrubText } from "./redact.js"; import { normalizeRoute } from "./fingerprint.js"; import { extractLlmCall } from "./llm.js"; import { @@ -10,7 +11,7 @@ import { type RuntimeSpanRow, } from "./types.js"; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; /** * OTLP/HTTP JSON → runtime signal. Structural types cover only the fields @@ -250,7 +251,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { traceId: span.traceId ?? "", spanId: span.spanId ?? "", parentSpanId: span.parentSpanId ?? null, - name: span.name ?? "unnamed", + name: scrubText(span.name ?? "unnamed").slice(0, 1000), kind, status: isError ? "error" : "ok", route: route ? normalizeRoute(route) : null, diff --git a/packages/otlp-ingester/src/redact.test.ts b/packages/otlp-ingester/src/redact.test.ts new file mode 100644 index 0000000..a25bffb --- /dev/null +++ b/packages/otlp-ingester/src/redact.test.ts @@ -0,0 +1,209 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { after, before, test } from "node:test"; +import type { IngesterConfig } from "./config.js"; +import { sanitizeRuntimeContext } from "./context.js"; +import { configureRedaction, scrubText } from "./redact.js"; +import { createIngesterApp } from "./server.js"; + +/** + * Server-side scrubbing is the backstop for senders that don't scrub (old + * SDKs, plain OTel SDKs in any language). Vectors are shared with the Node + * SDK, browser SDK and Python adapter (test-vectors/redaction.json). + */ + +interface Vector { + name: string; + input: string | string[]; + expect: string | string[]; + skip?: string[]; +} +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +) as { text: Vector[]; keys: { sensitive: string[]; safe: string[] } }; +const join = (value: string | string[]) => (Array.isArray(value) ? value.join("") : value); + +for (const vector of vectors.text) { + if (vector.skip?.includes("ingester")) continue; + test(`scrubText vector: ${vector.name}`, () => { + assert.equal(scrubText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive keys are masked at any depth; safe keys survive", () => { + for (const key of vectors.keys.sensitive) { + const out = sanitizeRuntimeContext({ outer: { list: [{ [key]: "raw" }] } }) as any; + assert.equal(out.outer.list[0][key], "[redacted]", key); + } + for (const key of vectors.keys.safe) { + const out = sanitizeRuntimeContext({ outer: { [key]: "kept" } }) as any; + assert.equal(out.outer[key], "kept", key); + } +}); + +test("operator-supplied patterns extend the built-ins", () => { + configureRedaction({ redactValuePatterns: ["EMP-\\d{4}"], redactKeyPatterns: ["^tenant_ref$"] }); + try { + assert.equal(scrubText("employee EMP-1234 and EMP-9999"), "employee [redacted] and [redacted]"); + assert.deepEqual(sanitizeRuntimeContext({ tenant_ref: "t1", plan: "pro" }), { + tenant_ref: "[redacted]", + plan: "pro", + }); + } finally { + configureRedaction({}); + } +}); + +// --------------------------------------------------------------------------- +// Over the wire: an unscrubbing OTLP sender → ingester → ClickHouse + sink. +// Nothing secret may reach either. +// --------------------------------------------------------------------------- + +let chStub: Server; +let sinkServer: Server; +let appServer: Server; +let ingestUrl: string; +const clickhouseBodies: string[] = []; +const sinkBodies: string[] = []; + +before(async () => { + chStub = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + clickhouseBodies.push(`${req.url}\n${body}`); + res.writeHead(200).end(); + }); + }); + await new Promise((resolve) => chStub.listen(0, resolve)); + sinkServer = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + sinkBodies.push(body); + res.writeHead(202).end(); + }); + }); + await new Promise((resolve) => sinkServer.listen(0, resolve)); + const config: IngesterConfig = { + port: 0, + clickhouseUrl: `http://127.0.0.1:${(chStub.address() as AddressInfo).port}`, + clickhouseUser: "default", + clickhousePassword: "", + clickhouseDatabase: "autter_runtime", + ingestKeys: [{ key: "test-key", orgId: "org-r", repositoryId: "repo-r" }], + keyValidatorUrl: null, + keyValidatorToken: null, + sinkUrl: `http://127.0.0.1:${(sinkServer.address() as AddressInfo).port}/sink`, + sinkToken: null, + sinkMaxAttempts: 3, + sinkMaxBufferedBatches: 10, + sinkMaxBufferedMb: 1, + maxBodyBytes: 1024 * 1024, + rateLimitPerMinute: 300, + clientRateLimitPerMinute: 120, + occurrenceTtlDays: 14, + spanTtlDays: 7, + metricsTtlDays: 90, + llmCallTtlDays: 90, + }; + const { app } = createIngesterApp(config); + appServer = app.listen(0); + await new Promise((resolve) => appServer.once("listening", resolve)); + ingestUrl = `http://127.0.0.1:${(appServer.address() as AddressInfo).port}`; +}); + +after(() => { + appServer.close(); + chStub.close(); + sinkServer.close(); +}); + +const str = (key: string, value: string) => ({ key, value: { stringValue: value } }); +const STRIPE = ["sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"].join(""); + +async function waitFor(check: () => boolean): Promise { + const deadline = Date.now() + 5000; + while (!check()) { + if (Date.now() > deadline) throw new Error("timed out"); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +test("secrets from an unscrubbing OTLP sender never reach ClickHouse or the sink", async () => { + const now = `${BigInt(Date.now()) * 1_000_000n}`; + const res = await fetch(`${ingestUrl}/v1/traces`, { + method: "POST", + headers: { authorization: "Bearer test-key", "content-type": "application/json" }, + body: JSON.stringify({ + resourceSpans: [{ + resource: { attributes: [str("service.name", "py-worker")] }, + scopeSpans: [{ + spans: [ + { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + name: "GET /reset/eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + kind: 2, + startTimeUnixNano: now, + endTimeUnixNano: now, + status: { code: 2, message: "psycopg2: postgres://admin:hunter2pg@db:5432/app" }, + attributes: [ + str("http.url", "https://api.test/cb?access_token=leakytoken99&page=2"), + str("http.request.header.cookie", "sid=rawcookie42"), + ], + events: [{ + name: "exception", + timeUnixNano: now, + attributes: [ + str("exception.type", "OperationalError"), + str("exception.message", `stripe rejected ${STRIPE} for jane@example.com`), + str("exception.stacktrace", 'Traceback (most recent call last):\n File "/app/pay.py", line 9, in charge\n stripe.api_key = "' + STRIPE + '"\nOperationalError: password=hunter3py'), + ], + }], + }, + { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b8", + name: "chat gpt-x", + kind: 3, + startTimeUnixNano: now, + endTimeUnixNano: now, + attributes: [ + str("gen_ai.system", "openai"), + str("gen_ai.request.model", "gpt-x"), + str("gen_ai.prompt", "summarise: my key is sk-proj-abcdefghijklmnopqrstuvwxyz0123"), + { key: "gen_ai.usage.input_tokens", value: { intValue: "321" } }, + str("enduser.id", "jane@example.com"), + ], + }, + ], + }], + }], + }), + }); + assert.equal(res.status, 200); + await waitFor(() => sinkBodies.length > 0); + + const everything = [...clickhouseBodies, ...sinkBodies].join("\n"); + for (const secret of [ + "hunter2pg", + "4eC39HqLyjWDarjtT1zdp7dc", + "jane@example.com", + "hunter3py", + "leakytoken99", + "rawcookie42", + "eyJhbGciOiJIUzI1NiJ9", + "abcdefghijklmnopqrstuvwxyz0123", + ]) { + assert.ok(!everything.includes(secret), `secret reached storage/sink: ${secret}`); + } + // Useful context survives: error type, scrubbed message shape, stack + // frames, and LLM token usage for cost tracking. + assert.ok(everything.includes("OperationalError")); + assert.ok(everything.includes("stripe rejected [redacted] for [redacted]")); + assert.ok(everything.includes('File \\"/app/pay.py\\", line 9, in charge')); + assert.match(everything, /321/); +}); diff --git a/packages/otlp-ingester/src/redact.ts b/packages/otlp-ingester/src/redact.ts new file mode 100644 index 0000000..c3aa926 --- /dev/null +++ b/packages/otlp-ingester/src/redact.ts @@ -0,0 +1,119 @@ +/** + * Server-side secret/PII scrubbing — the ingester's second line of defence. + * + * The SDKs scrub before anything leaves the customer's process, but the + * ingester also accepts older SDK versions and arbitrary third-party OTLP + * senders (any OTel SDK in any language) that scrub nothing. Everything this + * module touches is written to ClickHouse and forwarded to the sink webhook, + * which is where error context is later read for LLM-generated fixes — so + * scrubbing here, before storage, covers every downstream consumer. + * + * Same patterns as redactText() in @autter/runtime-node; parity is enforced + * by the shared test-vectors/redaction.json. + */ + +import type { RuntimeOccurrenceInput } from "./types.js"; + +const MASK = "[redacted]"; + +const PRIVATE_KEY_BLOCK_RE = + /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; +const PREFIXED_SECRET_RE = + /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g; +const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g; +const AUTH_SCHEME_RE = + /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +const HEADER_VALUE_RE = + /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +const SECRET_ASSIGNMENT_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +const URL_CREDENTIALS_RE = + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = + /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g; +const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; + +/** Keys whose values are masked wholesale (tested case-insensitively). */ +export const SENSITIVE_KEY_RE = + /password|passwd|passphrase|(^|[._-])pwd$|secret|token|credential|authori[sz]ation|^auth(-|_|$)|bearer|cookie|e-?mail|phone|(^|[^a-z])ssn($|[^a-z])|cvv|card[._-]?number|connection[._-]?string|(^|[._-])dsn$|(api|access|private|client|signing|encryption)[._-]?key|apikey|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid|request[._-]?body|response[._-]?body|headers|url[._-]?query/i; + +let extraValuePatterns: RegExp[] = []; +let extraKeyPatterns: RegExp[] = []; + +/** + * Deployment-specific patterns on top of the built-ins (env + * AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS, JSON arrays of + * regex sources). Server-side scrubbing has no off switch: it is the + * backstop for senders that don't scrub. + */ +export function configureRedaction(options: { + redactValuePatterns?: string[]; + redactKeyPatterns?: string[]; +}): void { + extraValuePatterns = (options.redactValuePatterns ?? []).map( + (source) => new RegExp(source, "gi"), + ); + extraKeyPatterns = (options.redactKeyPatterns ?? []).map( + (source) => new RegExp(source, "i"), + ); +} + +export function isSensitiveKey(key: string): boolean { + return SENSITIVE_KEY_RE.test(key) || extraKeyPatterns.some((re) => re.test(key)); +} + +function luhnValid(candidate: string): boolean { + const digits = candidate.replace(/\D/g, ""); + if (digits.length < 13 || digits.length > 19) return false; + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = digits.charCodeAt(digits.length - 1 - i) - 48; + if (i % 2 === 1) { + d *= 2; + if (d > 9) d -= 9; + } + sum += d; + } + return sum % 10 === 0; +} + +/** Scrub secrets/PII embedded in one string; the rest of it survives. */ +export function scrubText(value: string): string { + if (typeof value !== "string" || value === "") return value; + let out = value + .replace(PRIVATE_KEY_BLOCK_RE, MASK) + .replace(PREFIXED_SECRET_RE, MASK) + .replace(JWT_RE, MASK) + .replace(AUTH_SCHEME_RE, MASK) + .replace(HEADER_VALUE_RE, (_m, head: string) => head + MASK) + .replace(SECRET_ASSIGNMENT_RE, (_m, key: string, sep: string, val: string) => + key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + MASK + val[0] : MASK), + ) + .replace(URL_CREDENTIALS_RE, (_m, scheme: string) => `${scheme}${MASK}@`) + .replace(CARD_RE, (m) => (luhnValid(m) ? MASK : m)) + .replace(EMAIL_RE, MASK); + for (const re of extraValuePatterns) { + re.lastIndex = 0; + out = out.replace(re, MASK); + } + return out; +} + +/** Nullable convenience for optional columns. */ +export function scrubOptional(value: string | null): string | null { + return value === null ? null : scrubText(value); +} + +/** Scrub the free-text fields of an occurrence (message, stack, route, + * error type). Returns a copy. */ +export function scrubOccurrence( + occurrence: RuntimeOccurrenceInput, +): RuntimeOccurrenceInput { + return { + ...occurrence, + errorType: scrubText(occurrence.errorType), + message: scrubText(occurrence.message), + stack: scrubOptional(occurrence.stack), + route: scrubOptional(occurrence.route), + }; +} diff --git a/packages/otlp-ingester/src/server.ts b/packages/otlp-ingester/src/server.ts index 24d94c4..50e2eea 100644 --- a/packages/otlp-ingester/src/server.ts +++ b/packages/otlp-ingester/src/server.ts @@ -28,6 +28,7 @@ import { decodeProfile } from "./profiles.js"; import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js"; import { validateSourceMap } from "./source-maps.js"; import { SinkForwarder } from "./sink.js"; +import { configureRedaction, scrubOccurrence } from "./redact.js"; import type { IngestContext, RuntimeOccurrence, @@ -42,6 +43,7 @@ export interface IngesterApp { } export function createIngesterApp(config: IngesterConfig): IngesterApp { + configureRedaction(config); const store = new ClickHouseStore(config); // Fingerprinted occurrences feed the consumer's issue grouping, metric // points feed the request/error-rate rollups, LLM calls feed spend @@ -175,7 +177,13 @@ export function createIngesterApp(config: IngesterConfig): IngesterApp { ctx: IngestContext, inputs: RuntimeOccurrenceInput[], ): RuntimeOccurrence[] { - return inputs.map((input, index) => { + return inputs.map((raw, index) => { + // Second line of defence, before fingerprinting, storage, and the + // sink (which feeds LLM fix generation): scrub secrets from the + // free-text fields old SDKs and third-party OTLP senders pass + // through verbatim. Attributes were already sanitised by the + // normalisers (sanitizeRuntimeContext). + const input = scrubOccurrence(raw); const fingerprint = fingerprintOccurrence(input); return { ...input, diff --git a/packages/runtime-browser/README.md b/packages/runtime-browser/README.md index de33b5f..f00676a 100644 --- a/packages/runtime-browser/README.md +++ b/packages/runtime-browser/README.md @@ -82,7 +82,8 @@ initAutterBrowser({ | `setUser(id)` | **Opaque id only** — never an email | | `setContext(ctx)` | Attached to subsequent events | | `flush()` | Force-send the queue (also runs on page hide/unload) | -| `redactContext(ctx)` | Mask obvious PII in a context bag (applied to every event automatically) | +| `redactContext(ctx)` | Mask secrets/PII in a context bag, nested (applied to every event automatically) | +| `scrubText(text)` | Mask secrets/PII inside a string (applied to every message, stack and name automatically) | ## Batching & delivery @@ -101,9 +102,22 @@ Browser and OS are a family plus major version. Filenames and script URLs are query-stripped. For CSP blocks, the directive, blocked resource origin, script path, and a short policy hash are retained. -Custom `context` is free-form, so it is scrubbed before send: values under -sensitive-looking keys (`email`, `password`, `token`, `secret`, `auth`, -`cookie`, `api_key`, `card_number`, …) are replaced with `[redacted]`, and -email-shaped substrings are masked inside ordinary string values. This -mirrors the server SDK's `redactAttributes`; the relay and ingester apply -the same rules as defense-in-depth. +Messages, stack traces and custom `context` (at any depth) are scrubbed +before send: values under sensitive-looking keys (`email`, `password`, +`token`, `secret`, `auth`, `cookie`, `session`, `api_key`, `card_number`, …) +are replaced with `[redacted]`, and secrets inside strings — JWTs, +`Bearer`/`Basic` credentials, `Cookie:`/`Authorization:` text, connection +strings with credentials, common API keys, `?token=`/`password=` +assignments, emails, card numbers — are masked in place. This mirrors the +server SDK's `redactText`/`redactAttributes`; the relay and ingester apply +the full server-side rule set again as defense-in-depth. + +Add your own patterns (or opt out, not recommended): + +```ts +initAutterBrowser({ + ..., + redact: { keys: /^internal_ref$/, values: [/ORD-\d{5}/g] }, + // redact: false, +}); +``` diff --git a/packages/runtime-browser/src/index.ts b/packages/runtime-browser/src/index.ts index 8f84a20..2177ab9 100644 --- a/packages/runtime-browser/src/index.ts +++ b/packages/runtime-browser/src/index.ts @@ -5,8 +5,8 @@ * - zero runtime dependencies, < 5 KB gzipped (CI-enforced) * - no OTel SDK, no console patching, no DOM recording, no offline storage * - privacy by construction: pathname-only routes, no cookies / form values / - * request bodies; query strings stripped everywhere; custom context is - * scrubbed for obvious PII (emails, sensitive keys) before send + * request bodies; query strings stripped everywhere; messages, stacks and + * (nested) custom context are scrubbed for secrets and PII before send * * Payload contract: `/v1/browser` version 1 of the Autter otlp-ingester, * normally reached through the customer's same-origin relay @@ -40,6 +40,14 @@ export interface AutterBrowserOptions { captureTimings?: boolean; /** Attach the last safe click or form action to failures (default true). */ captureActions?: boolean; + /** + * Secret/PII scrubbing of messages, stacks, and context (on by default). + * Add your own patterns with `{ keys, values }` — `keys` masks whole + * context values whose key matches, `values` (use the `g` flag) masks + * matching substrings anywhere. `false` disables (not recommended; the + * relay and ingester still scrub). + */ + redact?: false | { keys?: RegExp; values?: RegExp[] }; } export type AutterSeverity = "fatal" | "error" | "warning" | "info"; @@ -117,26 +125,75 @@ function stripQuery(value: string | undefined): string | undefined { return value ? value.split("?")[0] : undefined; } -// Mini redaction — the browser twin of redactAttributes() in -// @autter/runtime-node. Custom context is free-form, so values under -// sensitive-looking keys and email-shaped strings are masked before -// anything leaves the page. Deliberately tiny: this bundle is size-capped. +// Mini redaction — the browser twin of redactText()/redactAttributes() in +// @autter/runtime-node (parity checked by test-vectors/redaction.json). +// Deliberately compact: this bundle is size-capped. const SENSITIVE_KEY_RE = - /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i; -const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; + /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|private[-_.]?key|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|(^|[^a-z])ssn($|[^a-z])|cvv|card([-_. ]?(number|num|no))?$/i; +const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; +// Vendor-prefixed keys, JWTs, Bearer/Basic credentials. Server-only shapes +// (PEM blocks, npm/GitLab/SendGrid tokens) are left to the relay/ingester. +const SECRET_RE = + /\b(sk-[\w-]{20,}|[sr]k_(live|test)_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[\w-]{30,}|(AKIA|ASIA)[0-9A-Z]{16}|eyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]{8,}|bearer\s+[\w.~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +const HEADER_RE = + /((^|[^\w-])(proxy-)?(authorization|(set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +const ASSIGN_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +const URL_CRED_RE = + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)([^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = /\b(4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(011|5\d{2}))([ -]?\d){9,15}\b/g; const MASK = "[redacted]"; +function luhn(value: string): boolean { + const digits = value.replace(/\D/g, ""); + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = +digits[digits.length - 1 - i]!; + if (i % 2) d = d * 2 > 9 ? d * 2 - 9 : d * 2; + sum += d; + } + return sum % 10 === 0; +} + +function redactOptions(): { keys?: RegExp; values?: RegExp[] } | false | undefined { + return opts ? opts.redact : undefined; +} + +/** Scrub secrets/PII embedded in one string (message, stack, URL, value). */ +export function scrubText(value: string): string { + const custom = redactOptions(); + if (custom === false) return value; + let out = String(value) + .replace(SECRET_RE, MASK) + .replace(HEADER_RE, "$1" + MASK) + .replace(ASSIGN_RE, (_m, key: string, sep: string, val: string) => + key + sep + (/^["']/.test(val) ? val[0] + MASK + val[0] : MASK)) + .replace(URL_CRED_RE, "$1" + MASK + "@") + .replace(CARD_RE, (m) => (luhn(m) ? MASK : m)) + .replace(EMAIL_RE, MASK); + for (const re of (custom && custom.values) || []) out = out.replace(re, MASK); + return out; +} + +function scrubValue(value: unknown, depth: number): unknown { + if (typeof value === "string") return scrubText(value); + if (!value || typeof value !== "object") return value; + if (depth > 5) return MASK; + if (Array.isArray(value)) return value.slice(0, 50).map((item) => scrubValue(item, depth + 1)); + return redactContext(value as Record, depth + 1); +} + export function redactContext( context: Record, + depth = 0, ): Record { + const custom = redactOptions(); + if (custom === false) return context; const out: Record = {}; for (const key in context) { - const value = context[key]; - out[key] = SENSITIVE_KEY_RE.test(key) + out[key] = SENSITIVE_KEY_RE.test(key) || (custom && custom.keys && custom.keys.test(key)) ? MASK - : typeof value === "string" - ? value.replace(EMAIL_RE, MASK) - : value; + : scrubValue(context[key], depth); } return out; } @@ -309,6 +366,9 @@ function enqueue(event: BrowserEvent, urgent?: boolean): void { } attachClient(event); // Scrub before beforeSend so the last-chance hook sees the final form. + event.message = scrubText(event.message); + if (event.stack) event.stack = scrubText(event.stack); + if (event.name) event.name = scrubText(event.name); if (event.context) event.context = redactContext(event.context); if (opts.beforeSend) { const mapped = opts.beforeSend(event); @@ -430,8 +490,8 @@ export function captureMessage( /** Report an application outcome that failed without throwing. Use a stable name. */ export function captureOutcome(name: string, message: string, context?: Record): void { - const event = baseEvent("outcome", String(message).replace(EMAIL_RE, MASK)); - event.name = String(name).replace(EMAIL_RE, MASK).slice(0, 200); + const event = baseEvent("outcome", message); + event.name = String(name).slice(0, 200); event.errorType = "OutcomeFailure"; event.severity = "error"; if (context) event.context = { ...(event.context || {}), ...context }; diff --git a/packages/runtime-browser/test/redact-e2e.test.mjs b/packages/runtime-browser/test/redact-e2e.test.mjs new file mode 100644 index 0000000..f25b916 --- /dev/null +++ b/packages/runtime-browser/test/redact-e2e.test.mjs @@ -0,0 +1,50 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { captureException, captureMessage, flush, initAutterBrowser, setContext } from "../dist/index.js"; + +test("messages, stacks and nested context are scrubbed before the beacon is sent", async () => { + const listeners = new Map(); + const sent = []; + globalThis.location = { href: "https://app.example.test/", pathname: "/", origin: "https://app.example.test" }; + globalThis.window = { fetch: async () => ({ status: 200 }), addEventListener(name, cb) { listeners.set(name, cb); } }; + globalThis.document = { addEventListener() {}, visibilityState: "visible", scripts: [] }; + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { userAgent: "", sendBeacon(_url, body) { sent.push(body); return true; } }, + }); + + initAutterBrowser({ + endpoint: "/api/autter-runtime", + service: "web", + sessionTracking: false, + captureTimings: false, + captureNetworkFailures: false, + captureActions: false, + // Customer-supplied patterns extend the built-ins. + redact: { keys: /^internal_ref$/, values: [/ORD-\d{5}/g] }, + }); + setContext({ checkout: { cardNumber: "4111111111111111", step: "pay" } }); + + const err = new Error("POST /login failed: {\"password\":\"hunter2\"} for jane@example.com"); + err.stack = `${err.name}: ${err.message}\n at submit (https://app.example.test/app.js:1:1)`; + captureException(err, { + internal_ref: "abc", + request: { headers: { authorization: "Bearer abcdefghijklmnop123" }, url: "/cb?access_token=tok123456" }, + }); + captureMessage("payment for ORD-12345 used token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM"); + listeners.get("unhandledrejection")({ reason: new Error("redis://:cachepw@cache:6379 down") }); + flush(); + + const body = await sent[0].text(); + for (const secret of ["hunter2", "jane@example.com", "4111111111111111", "abcdefghijklmnop123", + "tok123456", "ORD-12345", "eyJhbGciOiJIUzI1NiJ9", "cachepw"]) { + assert.ok(!body.includes(secret), `leaked: ${secret}`); + } + const { events } = JSON.parse(body); + assert.equal(events[0].message, 'POST /login failed: {"password":"[redacted]"} for [redacted]'); + assert.match(events[0].stack, /at submit \(https:\/\/app\.example\.test\/app\.js:1:1\)/); + assert.equal(events[0].context.internal_ref, "[redacted]"); + assert.equal(events[0].context.checkout.step, "pay"); + assert.equal(events[0].context.request.url, "/cb?access_token=[redacted]"); + assert.equal(events[2].message, "redis://[redacted]@cache:6379 down"); +}); diff --git a/packages/runtime-browser/test/redact-vectors.test.mjs b/packages/runtime-browser/test/redact-vectors.test.mjs new file mode 100644 index 0000000..41ac92f --- /dev/null +++ b/packages/runtime-browser/test/redact-vectors.test.mjs @@ -0,0 +1,38 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { redactContext, scrubText } from "../dist/index.js"; + +// Shared parity vectors — same file the Node SDK, ingester and Python +// adapter test against. `skip: ["browser"]` marks server-only shapes the +// size-capped browser bundle deliberately leaves to the relay/ingester. +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +); +const join = (value) => (Array.isArray(value) ? value.join("") : value); +const MASK = "[redacted]"; + +for (const vector of vectors.text) { + if (vector.skip?.includes("browser")) continue; + test(`scrubText vector: ${vector.name}`, () => { + assert.equal(scrubText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive context keys are masked at any depth; safe keys survive", () => { + for (const key of vectors.keys.sensitive) { + const out = redactContext({ outer: { list: [{ [key]: "raw" }] } }); + assert.equal(out.outer.list[0][key], MASK, key); + } + for (const key of vectors.keys.safe) { + assert.equal(redactContext({ outer: { [key]: "kept" } }).outer[key], "kept", key); + } +}); + +test("nested string values are scrubbed for secrets, not only emails", () => { + const out = redactContext({ + request: { url: "/api?token=abc123xyz&page=1", meta: ["Bearer abcdefghijklmnop12"] }, + }); + assert.equal(out.request.url, `/api?token=${MASK}&page=1`); + assert.deepEqual(out.request.meta, [MASK]); +}); diff --git a/packages/runtime-next/src/server.ts b/packages/runtime-next/src/server.ts index 4f8955a..4687e20 100644 --- a/packages/runtime-next/src/server.ts +++ b/packages/runtime-next/src/server.ts @@ -56,6 +56,7 @@ export { makeSafeCapture, installAutterAutoFlush, redactAttributes, + redactText, } from "@autter/runtime-node"; export type { diff --git a/packages/runtime-node/README.md b/packages/runtime-node/README.md index 1403b01..35d68a3 100644 --- a/packages/runtime-node/README.md +++ b/packages/runtime-node/README.md @@ -155,23 +155,39 @@ default 1% sampling the span contribution is negligible; if you set Tail-retained error traces don't distort this: their spans carry `autter.tail_retained` and the ingester keeps them out of span-fed rollups. -### Privacy: custom attributes are redacted before they leave the process - -The browser relay whitelist-sanitises everything a client posts — but the -server tracker accepts free-form attributes from *your* code, where a stray -`captureException(err, { "user.email": … })` would otherwise go out -verbatim. Custom attributes are therefore scrubbed at capture time by -default: - -- values that look like emails, JWTs, `sk-…`/`ghp_…`/AWS/Slack tokens, - `Bearer …` headers, or `scheme://user:pass@host` URLs are masked; +### Privacy: secrets and PII are scrubbed before they leave the process + +Error context is where secrets leak: `connect ECONNREFUSED +postgres://admin:hunter2@db…` in an exception message, a provider error +echoing an API key, a `?token=` in a request URL, a stray +`captureException(err, { "user.email": … })`. Everything the SDK exports is +therefore scrubbed by default — custom attributes, exception messages and +stack traces, span status messages, LLM call attributes and errors, +structured logs, and the browser events the relay forwards. A final pass at +export time scrubs every span regardless of who created it (HTTP +instrumentation URLs, third-party instrumentations, `recordException` in +your own code). + +- secrets inside any string are masked in place, keeping the rest of the + message/stack readable: JWTs, `Bearer`/`Basic` credentials, + `Authorization:`/`Cookie:`/`Set-Cookie:` header text, + `scheme://user:pass@host` connection strings (postgres, mysql, + mongodb+srv, redis, …), vendor keys (`sk-…`, `sk_live_…`, `ghp_…`, + `github_pat_…`, `xox?-…`, `AIza…`, `AKIA…`, …), PEM private keys, + `password=`/`?token=`/`?api_key=`-style assignments, emails, and + Luhn-valid card numbers; - attributes whose **key** looks sensitive (`password`, `token`, `secret`, - `api_key`, `authorization`, `cookie`, `ssn`, `card_number`, …) are masked - wholesale; -- non-sensitive keys and primitives pass through untouched, so grouping and - dashboards keep working. + `api_key`, `authorization`, `cookie`, `session`, `ssn`, `card_number`, …) + are masked wholesale, at any nesting depth; +- non-sensitive keys, numbers (including LLM token counts) and booleans pass + through untouched, so grouping, dashboards and cost tracking keep working. + +The same patterns run in the browser SDK, the relay, the Python adapter, +and again in the ingester before storage (shared test vectors in +`test-vectors/redaction.json`). -Disable or extend it per service: +Extend it (applies to messages, stacks and URLs too) or disable it per +service: ```ts initAutterServer({ @@ -193,7 +209,9 @@ const safe = makeSafeCapture(); safe.captureException(err, { "user.email": email }); // masked before export ``` -The raw primitive is exported too (`redactAttributes(attrs, options)`). +The raw primitives are exported too (`redactAttributes(attrs, options)`, +`redactText(text, options)`). The browser relay takes the same options as +`createBrowserRelayHandler({ ..., redact })`. This closes the server-side gap to match the browser relay's payload whitelist; it is best-effort scrubbing of obvious PII shapes, not a DLP engine — keep secrets out of attributes in the first place. diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index bc3603f..90804f8 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -31,6 +31,7 @@ export { } from "./lifecycle.js"; export { redactAttributes, + redactText, type RedactOptions, } from "./redact.js"; export { diff --git a/packages/runtime-node/src/llm-instrument.ts b/packages/runtime-node/src/llm-instrument.ts index 849be01..45bcf65 100644 --- a/packages/runtime-node/src/llm-instrument.ts +++ b/packages/runtime-node/src/llm-instrument.ts @@ -4,7 +4,7 @@ import { type Attributes, type Span, } from "@opentelemetry/api"; -import { autterLlmTracer } from "./server.js"; +import { autterLlmTracer, recordError, redactWithActive } from "./server.js"; /** * One-line LLM client auto-instrumentation: @@ -115,11 +115,9 @@ function applyUsage(span: Span, usage: UsageShape): void { } function endWithError(span: Span, err: unknown): void { - if (err instanceof Error) span.recordException(err); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + // Provider errors echo request details (API key fragments, prompt + // snippets, account emails) — recorded with message/stack scrubbed. + recordError(span, err); span.end(); } @@ -223,7 +221,7 @@ export function instrumentLlmClient( "gen_ai.system": provider, "gen_ai.request.model": request.model, ...(userId ? { "autter.user_id": userId } : {}), - ...options?.attributes, + ...redactWithActive(options?.attributes), }, }); let result: unknown; diff --git a/packages/runtime-node/src/redact-exporter.ts b/packages/runtime-node/src/redact-exporter.ts new file mode 100644 index 0000000..2f0a956 --- /dev/null +++ b/packages/runtime-node/src/redact-exporter.ts @@ -0,0 +1,81 @@ +import type { ExportResult } from "@opentelemetry/core"; +import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; +import type { Redactor } from "./redact.js"; + +/** + * Last line of defence inside the customer's process: every span is + * scrubbed on its way to the OTLP exporter, whoever produced it. Capture + * calls (captureException, withLlmCall, …) already redact what they write, + * but spans also carry data the SDK never sees at capture time: + * + * - `span.recordException(err)` from HttpInstrumentation, the customer's + * own code, or third-party instrumentations — raw message + stack; + * - `http.url` / `http.target` / `url.full` with `?token=` query strings; + * - request headers when the host enables `headersToSpanAttributes`; + * - span status messages copied from `err.message`. + * + * The span is re-wrapped, never mutated (spans are shared with other + * processors). Disabled when the host passes `redactAttributes: false`. + */ +export function redactSpan(span: ReadableSpan, redactor: Redactor): ReadableSpan { + const status = span.status.message + ? { ...span.status, message: redactor.text(span.status.message) } + : span.status; + return { + name: redactor.text(span.name), + kind: span.kind, + spanContext: () => span.spanContext(), + parentSpanId: span.parentSpanId, + startTime: span.startTime, + endTime: span.endTime, + status, + attributes: redactor.exported(span.attributes), + links: span.links, + events: span.events.map((event) => + event.attributes + ? { ...event, attributes: redactor.exported(event.attributes) } + : event, + ), + duration: span.duration, + ended: span.ended, + resource: span.resource, + instrumentationLibrary: span.instrumentationLibrary, + droppedAttributesCount: span.droppedAttributesCount, + droppedEventsCount: span.droppedEventsCount, + droppedLinksCount: span.droppedLinksCount, + }; +} + +export class RedactingSpanExporter implements SpanExporter { + constructor( + private readonly inner: SpanExporter, + private readonly redactor: () => Redactor, + ) {} + + export( + spans: ReadableSpan[], + resultCallback: (result: ExportResult) => void, + ): void { + const redactor = this.redactor(); + let scrubbed = spans; + if (redactor.enabled) { + scrubbed = []; + for (const span of spans) { + try { + scrubbed.push(redactSpan(span, redactor)); + } catch { + // Fail closed: a span we could not scrub is not exported. + } + } + } + this.inner.export(scrubbed, resultCallback); + } + + shutdown(): Promise { + return this.inner.shutdown(); + } + + forceFlush(): Promise { + return this.inner.forceFlush?.() ?? Promise.resolve(); + } +} diff --git a/packages/runtime-node/src/redact.ts b/packages/runtime-node/src/redact.ts index bd93438..e2bfec4 100644 --- a/packages/runtime-node/src/redact.ts +++ b/packages/runtime-node/src/redact.ts @@ -1,12 +1,25 @@ import type { Attributes } from "@opentelemetry/api"; /** - * Server-side attribute redaction. The browser relay whitelist-sanitises - * everything a client posts (see relay.ts) — but custom attributes handed to - * `captureException` / `captureMessage` left the process verbatim. This module - * closes that gap: emails, tokens, API keys, and credentials embedded in - * string values are masked, and attributes whose NAME looks sensitive are - * masked wholesale, before anything is exported. + * Server-side secret/PII redaction. Applied to everything the Node SDK + * exports: custom attributes, exception messages and stack traces, span + * status messages, span/event attributes set by instrumentations (HTTP URLs + * with query strings, third-party instrumentations), and structured logs. + * + * Two mechanisms: + * - KEY patterns: an attribute whose NAME looks sensitive (password, token, + * cookie, authorization, api_key, session, ssn, …) is masked wholesale, + * at any nesting depth. + * - VALUE patterns: secrets embedded inside any string (JWTs, bearer/basic + * credentials, Cookie/Authorization header text, `scheme://user:pass@` + * connection strings, vendor API keys, PEM private keys, `password=` / + * `?token=` assignments, emails, Luhn-valid card numbers) are replaced in + * place so the rest of the string — the useful part of an error message + * or stack frame — survives. + * + * The same pattern set is mirrored (and kept in parity by the shared + * test-vectors/redaction.json) in @autter/runtime-browser, the ingester, and + * adapters/python/redact.py. */ type AttrValue = @@ -24,8 +37,10 @@ export interface RedactOptions { */ additionalKeyPatterns?: (RegExp | string)[]; /** - * Extra patterns scrubbed inside string VALUES (same semantics as the - * built-in email/token patterns). Extends the built-in list. + * Extra patterns scrubbed inside string VALUES — attribute values, + * exception messages, stack traces, span status messages, URLs, log + * lines. Strings are compiled as case-insensitive regex sources. Every + * pattern is applied globally. Extends the built-in list. */ additionalValuePatterns?: (RegExp | string)[]; /** Replacement token. Default "[redacted]". */ @@ -35,50 +50,89 @@ export interface RedactOptions { * sensitive keys. Default true. */ scrubEmailValues?: boolean; + /** + * Scrub Luhn-valid payment card numbers from all string values. + * Default true. + */ + scrubCardNumbers?: boolean; } // Tested against the lower-cased KEY. Deliberately anchored where a loose // substring would over-redact ("card" must not eat "discard", -// "author" must not eat "author_id"). +// "author" must not eat "author_id", "session_id" is an attribution id). const SENSITIVE_KEY_PATTERNS: RegExp[] = [ /e-?mail/, - /pass(word|wd|phrase)|^pass$/, + /pass(word|wd|phrase)|^pass$|(^|[._-])pwd$/, /token/, /secret/, /credential/, /(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key/, - /^-?x?-?authorization$|^auth(-|_|$)|bearer/, + /authori[sz]ation|^auth(-|_|$)|bearer/, /cookie/, + /(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid/, /phone|msisdn/, - /\bssn\b|social[-_ ]?security/, + /(^|[^a-z])ssn($|[^a-z])|social[-_ ]?security/, /cvv|cvc|card([-_. ]?(number|num|no))?$/, /credit[-_.]?card/, - /connection[-_.]?string/, + /connection[-_.]?string|(^|[._-])dsn$/, /recovery[-_.]?code|\botp\b|magic[-_.]?link/, ]; -// Scrubbed INSIDE string values (matched substrings are replaced, the rest -// of the value survives — useful context like a stack frame stays readable). +// --------------------------------------------------------------------------- +// VALUE patterns — scrubbed INSIDE strings. Order matters: whole-token +// shapes first, then header/assignment context, then URLs, cards, emails. +// Every pattern is idempotent on its own mask ("[redacted]" never matches). +// --------------------------------------------------------------------------- + const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; -const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; -const OPENAI_KEY_RE = /\bsk-[A-Za-z0-9]{20,}\b/g; -const GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g; -const AWS_KEY_RE = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g; -const SLACK_TOKEN_RE = /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g; -const BEARER_RE = /\bbearer\s+[A-Za-z0-9._~+/=-]{10,}/gi; -// postgres://user:password@host — credentials gone, host kept. +/** Vendor credentials recognisable by prefix alone. */ +const PREFIXED_SECRET_RE = + /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g; +const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g; +/** `Bearer ` and `Basic ` (the latter must look encoded, so + * the phrase "basic authentication" survives). */ +const AUTH_SCHEME_RE = + /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +/** `Authorization: …`, `Cookie: …`, `Set-Cookie: …` (raw header text, JSON, + * util.inspect): the whole value goes, since cookies and auth schemes + * contain spaces and semicolons. */ +const HEADER_VALUE_RE = + /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +/** `password=…`, `"token":"…"`, `?api_key=…&` — a secret-named key followed + * by `=` or `:`. Value ends at a delimiter so the surrounding query string + * or JSON stays readable. */ +const SECRET_ASSIGNMENT_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +/** `scheme://user:pass@host` (user may be empty, pass may contain `@`) and + * long token-as-username URLs (`https://@host`). Host is kept. */ const URL_CREDENTIALS_RE = - /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi; + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = + /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g; +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; + +function luhnValid(candidate: string): boolean { + const digits = candidate.replace(/\D/g, ""); + if (digits.length < 13 || digits.length > 19) return false; + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = digits.charCodeAt(digits.length - 1 - i) - 48; + if (i % 2 === 1) { + d *= 2; + if (d > 9) d -= 9; + } + sum += d; + } + return sum % 10 === 0; +} interface CompiledRedactor { keyPatterns: RegExp[]; - valuePatterns: RegExp[]; - urlCredentials: RegExp; - privateKeyBlock: RegExp; + extraValuePatterns: RegExp[]; mask: string; scrubEmailValues: boolean; + scrubCardNumbers: boolean; } function toCaseInsensitive(pattern: RegExp | string): RegExp { @@ -87,46 +141,62 @@ function toCaseInsensitive(pattern: RegExp | string): RegExp { : new RegExp(pattern.source, pattern.flags.includes("i") ? pattern.flags : `${pattern.flags}i`); } +/** Value patterns must be global, or String#replace masks only the first hit. */ +function toGlobal(pattern: RegExp | string): RegExp { + if (typeof pattern === "string") return new RegExp(pattern, "gi"); + return pattern.global + ? pattern + : new RegExp(pattern.source, `${pattern.flags}g`); +} + function compile(options?: RedactOptions): CompiledRedactor { const keyPatterns = [...SENSITIVE_KEY_PATTERNS]; - const valuePatterns: RegExp[] = [ - JWT_RE, - OPENAI_KEY_RE, - GITHUB_TOKEN_RE, - AWS_KEY_RE, - SLACK_TOKEN_RE, - BEARER_RE, - ]; - if (options?.additionalKeyPatterns) { - for (const p of options.additionalKeyPatterns) { - keyPatterns.push(toCaseInsensitive(p)); - } + const extraValuePatterns: RegExp[] = []; + for (const p of options?.additionalKeyPatterns ?? []) { + keyPatterns.push(toCaseInsensitive(p)); } - if (options?.additionalValuePatterns) { - for (const p of options.additionalValuePatterns) { - if (!valuePatterns.some((existing) => existing.source === (typeof p === "string" ? p : p.source))) { - valuePatterns.push(typeof p === "string" ? new RegExp(p, "gi") : p); - } + for (const p of options?.additionalValuePatterns ?? []) { + const re = toGlobal(p); + if (!extraValuePatterns.some((existing) => existing.source === re.source)) { + extraValuePatterns.push(re); } } return { keyPatterns, - valuePatterns, - urlCredentials: URL_CREDENTIALS_RE, - privateKeyBlock: PRIVATE_KEY_BLOCK_RE, + extraValuePatterns, mask: options?.mask ?? "[redacted]", scrubEmailValues: options?.scrubEmailValues !== false, + scrubCardNumbers: options?.scrubCardNumbers !== false, }; } +/** Hard cap on how much of one string is scanned: stacks are capped well + * below this upstream; anything longer is truncated rather than exported + * unscanned. */ +const MAX_SCRUB_CHARS = 64 * 1024; + function redactString(value: string, r: CompiledRedactor): string { - let out = value.replace(r.privateKeyBlock, r.mask); - for (const re of r.valuePatterns) { - out = out.replace(re, r.mask); + const mask = r.mask; + let out = value.length > MAX_SCRUB_CHARS ? value.slice(0, MAX_SCRUB_CHARS) : value; + out = out + .replace(PRIVATE_KEY_BLOCK_RE, mask) + .replace(PREFIXED_SECRET_RE, mask) + .replace(JWT_RE, mask) + .replace(AUTH_SCHEME_RE, mask) + .replace(HEADER_VALUE_RE, (_m, head: string) => head + mask) + .replace(SECRET_ASSIGNMENT_RE, (_m, key: string, sep: string, val: string) => + key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + mask + val[0] : mask), + ) + .replace(URL_CREDENTIALS_RE, (_m, scheme: string) => `${scheme}${mask}@`); + if (r.scrubCardNumbers) { + out = out.replace(CARD_RE, (m) => (luhnValid(m) ? mask : m)); } - out = out.replace(r.urlCredentials, "$1" + r.mask + "@"); if (r.scrubEmailValues) { - out = out.replace(EMAIL_VALUE_RE, r.mask); + out = out.replace(EMAIL_VALUE_RE, mask); + } + for (const re of r.extraValuePatterns) { + re.lastIndex = 0; + out = out.replace(re, mask); } return out; } @@ -393,16 +463,74 @@ function redactWith( return out; } +/** + * Scrub secrets/PII embedded in one string — an error message, a stack + * trace, a URL, a log line. Same value patterns as redactAttributes(). + */ +export function redactText(text: string, options?: RedactOptions): string { + return typeof text === "string" ? redactString(text, compile(options)) : text; +} + +/** A compiled redactor: callable for attribute bags, plus string and + * export-time helpers. */ +export interface Redactor { + (attributes?: Attributes | null): Attributes; + /** Scrub one string (messages, stacks, URLs). Identity when disabled. */ + text(value: string): string; + /** + * Export-time pass over attributes already on a span (set by any + * instrumentation, not just Autter's capture calls). Strings are + * value-scrubbed, string values under sensitive KEYS are masked, and + * numbers/booleans pass untouched — usage counters such as + * `ai.usage.promptTokens` must survive for cost tracking. + */ + exported(attributes: Attributes): Attributes; + readonly enabled: boolean; +} + /** * Compile a redactor once for a hot path — initAutterServer builds one from * its options and reuses it for every capture instead of recompiling. + * `false` returns a pass-through (redaction disabled by the host). */ -export function makeRedactor( - options?: boolean | RedactOptions, -): (attributes?: Attributes | null) => Attributes { +export function makeRedactor(options?: boolean | RedactOptions): Redactor { if (options === false) { - return (attributes) => ({ ...(attributes ?? {}) }); + const passthrough = ((attributes?: Attributes | null) => ({ + ...(attributes ?? {}), + })) as Redactor; + passthrough.text = (value) => value; + passthrough.exported = (attributes) => attributes; + (passthrough as { enabled: boolean }).enabled = false; + return passthrough; } const r = compile(options === true ? undefined : options); - return (attributes) => redactWith(attributes, r); + const redactor = ((attributes?: Attributes | null) => + redactWith(attributes, r)) as Redactor; + redactor.text = (value) => + typeof value === "string" ? redactString(value, r) : value; + redactor.exported = (attributes) => { + const out: Attributes = {}; + for (const key of Object.keys(attributes)) { + const value = attributes[key]; + if (typeof value === "string") { + out[key] = r.keyPatterns.some((re) => re.test(key.toLowerCase())) + ? r.mask + : redactString(value, r); + } else if (Array.isArray(value)) { + const sensitive = r.keyPatterns.some((re) => re.test(key.toLowerCase())); + out[key] = (value as unknown[]).map((item) => + typeof item === "string" + ? sensitive + ? r.mask + : redactString(item, r) + : item, + ) as typeof value; + } else { + out[key] = value; + } + } + return out; + }; + (redactor as { enabled: boolean }).enabled = true; + return redactor; } diff --git a/packages/runtime-node/src/relay.ts b/packages/runtime-node/src/relay.ts index 3eb07d5..0482152 100644 --- a/packages/runtime-node/src/relay.ts +++ b/packages/runtime-node/src/relay.ts @@ -1,4 +1,6 @@ import type { IncomingMessage, ServerResponse } from "node:http"; +import type { Attributes } from "@opentelemetry/api"; +import { makeRedactor, type RedactOptions, type Redactor } from "./redact.js"; /** * Same-origin browser relay. The browser tracker posts to a route on the @@ -23,6 +25,15 @@ export interface RelayOptions { perIpRateLimit?: number | false; /** Called when the async forward fails (default: console.warn). */ onError?: (err: unknown) => void; + /** + * Scrub secrets/PII from browser events (message, stack, name, nested + * context) before they leave your server — the same patterns as + * `initAutterServer({ redactAttributes })`. Catches old browser SDK + * versions and anything the browser's lean redactor misses. Pass a + * RedactOptions object to add your own patterns; `false` disables. + * Default true. + */ + redact?: boolean | RedactOptions; } class IpWindow { @@ -69,7 +80,12 @@ const SEVERITIES = new Set(["fatal", "error", "warning", "info"]); // Whitelist sanitiser — anything not listed here is dropped, so a // compromised or buggy client can't smuggle cookies/DOM/bodies through the // relay. Returns null when the payload is structurally invalid. -export function sanitizeBrowserPayload(raw: unknown): object | null { +const defaultRelayRedactor = makeRedactor(true); + +export function sanitizeBrowserPayload( + raw: unknown, + redactor: Redactor = defaultRelayRedactor, +): object | null { if (typeof raw !== "object" || raw === null) return null; const p = raw as Record; if (p.version !== 1) return null; @@ -90,12 +106,16 @@ export function sanitizeBrowserPayload(raw: unknown): object | null { ? { severity: e.severity } : {}), message: - typeof e.message === "string" ? e.message.slice(0, 4000) : "", - ...(typeof e.name === "string" ? { name: e.name.slice(0, 200) } : {}), + typeof e.message === "string" + ? redactor.text(e.message.slice(0, 4000)) + : "", + ...(typeof e.name === "string" + ? { name: redactor.text(e.name.slice(0, 200)) } + : {}), ...(typeof e.durationMs === "number" && Number.isFinite(e.durationMs) ? { durationMs: Math.max(0, Math.min(120000, e.durationMs)) } : {}), ...(typeof e.stack === "string" - ? { stack: e.stack.slice(0, 32000) } + ? { stack: redactor.text(e.stack.slice(0, 32000)) } : {}), ...(typeof e.errorType === "string" ? { errorType: e.errorType.slice(0, 200) } @@ -108,8 +128,10 @@ export function sanitizeBrowserPayload(raw: unknown): object | null { ...(typeof e.route === "string" ? { route: e.route.split("?")[0]!.slice(0, 1000) } : {}), - ...(typeof e.context === "object" && e.context !== null - ? { context: e.context } + ...(typeof e.context === "object" && + e.context !== null && + !Array.isArray(e.context) + ? { context: redactor(e.context as Attributes) } : {}), }); } @@ -154,6 +176,10 @@ export function createBrowserRelayFetchHandler( opts: RelayOptions, ): (request: Request) => Promise { const maxBody = opts.maxBodyBytes ?? DEFAULT_MAX_BODY; + const redactor = + opts.redact === undefined || opts.redact === true + ? defaultRelayRedactor + : makeRedactor(opts.redact); const limiter = opts.perIpRateLimit === false ? null @@ -185,7 +211,7 @@ export function createBrowserRelayFetchHandler( status: 400, }); } - const payload = sanitizeBrowserPayload(raw); + const payload = sanitizeBrowserPayload(raw, redactor); if (!payload) { return new Response(JSON.stringify({ error: "invalid payload" }), { status: 400, @@ -211,6 +237,10 @@ export function createBrowserRelayHandler( res: ServerResponse, ) => void { const maxBody = opts.maxBodyBytes ?? DEFAULT_MAX_BODY; + const redactor = + opts.redact === undefined || opts.redact === true + ? defaultRelayRedactor + : makeRedactor(opts.redact); const limiter = opts.perIpRateLimit === false ? null @@ -227,7 +257,7 @@ export function createBrowserRelayHandler( } function handleParsed(raw: unknown, res: ServerResponse): void { - const payload = sanitizeBrowserPayload(raw); + const payload = sanitizeBrowserPayload(raw, redactor); if (!payload) { respond(res, 400, { error: "invalid payload" }); return; diff --git a/packages/runtime-node/src/server.ts b/packages/runtime-node/src/server.ts index 883b1a4..9bfc348 100644 --- a/packages/runtime-node/src/server.ts +++ b/packages/runtime-node/src/server.ts @@ -55,7 +55,9 @@ import { makeRedactor, redactAttributes, type RedactOptions, + type Redactor, } from "./redact.js"; +import { RedactingSpanExporter } from "./redact-exporter.js"; import { startMemoryMetrics } from "./memory.js"; import { configureRuntimeLogger, flushRuntimeLogs, shutdownRuntimeLogger, type RuntimeLoggingOptions } from "./logger.js"; @@ -125,12 +127,17 @@ export interface AutterServerOptions { */ autoFlush?: boolean; /** - * Mask PII/secrets in custom attributes before they leave the process: - * emails, tokens (JWT, sk-, gh_, AWS, Slack), bearer headers, - * `scheme://user:pass@` URLs, and anything whose attribute KEY looks - * sensitive (password/token/secret/cookie/…). Mirrors the browser - * relay's whitelist sanitiser. `true`/undefined = defaults; pass a - * RedactOptions object to extend the patterns; `false` disables. + * Mask secrets/PII before anything leaves the process — custom + * attributes, exception messages and stack traces, span status + * messages, URLs/query strings and attributes set by any + * instrumentation (scrubbed again at export), and structured logs: + * JWTs, bearer/basic credentials, Cookie/Authorization header text, + * `scheme://user:pass@` connection strings, vendor API keys (sk-, sk_live_, + * ghp_, xox*-, AIza, AKIA, …), PEM private keys, `password=`/`?token=` + * assignments, emails, card numbers, and anything whose attribute KEY + * looks sensitive (password/token/secret/cookie/authorization/session/…). + * `true`/undefined = defaults; pass a RedactOptions object to add your + * own key/value patterns; `false` disables (not recommended). * Default true. */ redactAttributes?: boolean | RedactOptions; @@ -585,7 +592,7 @@ async function runLlmSpan( setCost: (usd) => span.setAttribute("autter.llm.cost_usd", usd), setResponseModel, setModel: setResponseModel, - setAttributes: (attributes) => span.setAttributes(attributes), + setAttributes: (attributes) => span.setAttributes(activeRedactor(attributes)), }; try { const result = await context.with( @@ -595,14 +602,8 @@ async function runLlmSpan( span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (err) { - if (err instanceof Error) { - span.recordException(err); - span.setAttribute("error.type", err.name); - } - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + recordError(span, err); + if (err instanceof Error) span.setAttribute("error.type", err.name); throw err; } finally { span.end(); @@ -622,15 +623,10 @@ function recordLlmCall(tracer: Tracer, call: TrackedLlmCall): void { }, ); if (call.error !== undefined && call.error !== null) { + recordError(span, call.error); if (call.error instanceof Error) { - span.recordException(call.error); span.setAttribute("error.type", call.error.name); } - span.setStatus({ - code: SpanStatusCode.ERROR, - message: - call.error instanceof Error ? call.error.message : String(call.error), - }); } else { span.setStatus({ code: SpanStatusCode.OK }); } @@ -652,17 +648,42 @@ async function runWithSpan( span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (err) { - if (err instanceof Error) span.recordException(err); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + recordError(span, err); throw err; } finally { span.end(); } } +/** + * Record a thrown value on a span with its message and stack scrubbed — + * the redacting replacement for `span.recordException(err)` + an error + * status. Same wire shape as OTel's recordException (an "exception" event + * with exception.type/message/stacktrace). + * @internal + */ +export function recordError(span: Span, err: unknown): void { + const isError = err instanceof Error; + const message = activeRedactor.text(isError ? err.message : String(err)); + // Like recordException, only real Errors get an exception event; other + // thrown values keep just the (scrubbed) error status. + if (isError) { + span.addEvent("exception", { + "exception.type": err.name, + "exception.message": message, + ...(err.stack + ? { "exception.stacktrace": activeRedactor.text(err.stack) } + : {}), + }); + } + span.setStatus({ code: SpanStatusCode.ERROR, message }); +} + +/** @internal Redact attributes with the active configuration. */ +export function redactWithActive(attributes?: Attributes | null): Attributes { + return activeRedactor(attributes); +} + /** Express (4/5) assigns routing state onto the core request object; a * matched handler leaves the route template on `req.route.path` and the * mount prefix on `req.baseUrl`. */ @@ -722,7 +743,7 @@ let activeAlwaysOnProvider: BasicTracerProvider | null = null; /** Compiled attribute redactor — defaults until initAutterServer applies its * own configuration. Used by every capture path including LLM attributes. */ -let activeRedactor = makeRedactor(true); +let activeRedactor: Redactor = makeRedactor(true); export function initAutterServer(options: AutterServerOptions): AutterServer { if (active) return active; @@ -732,6 +753,16 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { "", ); const headers = { authorization: `Bearer ${options.apiKey}` }; + // Every trace exporter scrubs spans on the way out, whoever created them + // (see redact-exporter.ts). Reads the active redactor lazily so the + // configuration below applies. + const traceExporter = () => + new CountingExporter( + new RedactingSpanExporter( + new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), + () => activeRedactor, + ), + ); const environment = options.environment ?? process.env.NODE_ENV ?? "production"; // Debug mode: option OR AUTTER_DEBUG env (lifecycle seeds itself from the @@ -771,22 +802,13 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { if (options.llmTracing !== false) sampler = new LlmAwareSampler(sampler); const errorTraceBuffer = retainTraces ? new ErrorTraceRetentionProcessor( - new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - { scheduledDelayMillis: 2000 }, - ), + new BatchSpanProcessor(traceExporter(), { scheduledDelayMillis: 2000 }), retainOnError, slowThresholdMs, ) : null; - const mainSpanProcessor = new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - ); + const mainSpanProcessor = new BatchSpanProcessor(traceExporter()); const metricReader = new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter({ url: `${endpoint}/v1/metrics`, @@ -831,12 +853,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { resource, sampler: new AlwaysOnSampler(), spanProcessors: [ - new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - { scheduledDelayMillis: 2000 }, - ), + new BatchSpanProcessor(traceExporter(), { scheduledDelayMillis: 2000 }), ], }); const errorTracer = alwaysOnProvider.getTracer("autter-errors"); @@ -852,9 +869,12 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { errorTraceBuffer?.retainActiveTrace(); telemetryStats.markCaptured(); const isError = error instanceof Error; - const message = isError ? error.message : String(error); - // Redaction is applied here, not at export time: PII must not leave - // the process even if an exporter misbehaves. + // Redaction is applied here AND at export time: secrets must not + // leave the process even if an exporter misbehaves. Messages and + // stacks are scrubbed too — `connect ECONNREFUSED + // postgres://admin:hunter2@…` is the typical leak, not attributes. + const message = activeRedactor.text(isError ? error.message : String(error)); + const stackText = isError && error.stack ? activeRedactor.text(error.stack) : ""; const redacted = activeRedactor(attributes); // Prefer the live request span (the HTTP server span inside an Express @@ -874,18 +894,15 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { : errorTracer.startSpan(isError ? error.name : "Error"); span.setAttributes({ "autter.severity": "error", ...redacted }); - if (isError && error.stack) { - if (redacted["autter.handled"] === true) { - span.addEvent("exception", { - "exception.type": error.name, - "exception.message": message, - "exception.stacktrace": error.stack, - "autter.handled": true, - "autter.sampled": redacted["autter.sampled"] === true, - }); - } else { - span.recordException(error); - } + if (isError && stackText) { + span.addEvent("exception", { + "exception.type": error.name, + "exception.message": message, + "exception.stacktrace": stackText, + ...(redacted["autter.handled"] === true + ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } + : {}), + }); } else { // No usable stack — an Error thrown without one, or a non-Error // value. Synthesize the call site (minus this frame) so the @@ -898,7 +915,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { span.addEvent("exception", { "exception.type": isError ? error.name : "Error", "exception.message": message, - ...(stack ? { "exception.stacktrace": stack } : {}), + ...(stack ? { "exception.stacktrace": activeRedactor.text(stack) } : {}), ...(redacted["autter.handled"] === true ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } : {}), @@ -929,16 +946,17 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { ?.split("\n") .filter((line, i) => i === 0 || !line.includes("captureMessage")) .join("\n"); + const safeMessage = activeRedactor.text(message); const span = errorTracer.startSpan("Message", { attributes: { "autter.severity": severity, ...activeRedactor(attributes) }, }); span.addEvent("exception", { "exception.type": "Message", - "exception.message": message, - ...(stack ? { "exception.stacktrace": stack } : {}), + "exception.message": safeMessage, + ...(stack ? { "exception.stacktrace": activeRedactor.text(stack) } : {}), "autter.severity": severity, }); - span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage }); span.end(); } @@ -1046,14 +1064,14 @@ export function captureException( active.captureException(error, attributes); return; } + // Before initAutterServer the span rides the host's global provider, + // whose exporter we don't control — so scrub here, at capture time. const span = trace .getTracer("autter-errors") - .startSpan(error instanceof Error ? error.name : "Error", { attributes }); - if (error instanceof Error) span.recordException(error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: error instanceof Error ? error.message : String(error), - }); + .startSpan(error instanceof Error ? error.name : "Error", { + attributes: activeRedactor(attributes), + }); + recordError(span, error); span.end(); } @@ -1068,7 +1086,12 @@ export function withProcessSpan( attributes?: Attributes, ): Promise { if (active) return active.withProcessSpan(name, fn, attributes); - return runWithSpan(trace.getTracer("autter-processes"), name, fn, attributes); + return runWithSpan( + trace.getTracer("autter-processes"), + name, + fn, + activeRedactor(attributes), + ); } /** @@ -1103,15 +1126,16 @@ export function captureMessage( active.captureMessage(message, severity, attributes); return; } + const safeMessage = activeRedactor.text(message); const span = trace.getTracer("autter-errors").startSpan("Message", { - attributes: { "autter.severity": severity, ...attributes }, + attributes: { "autter.severity": severity, ...activeRedactor(attributes) }, }); span.addEvent("exception", { "exception.type": "Message", - "exception.message": message, + "exception.message": safeMessage, "autter.severity": severity, }); - span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage }); span.end(); } diff --git a/packages/runtime-node/test/autoflush.test.mjs b/packages/runtime-node/test/autoflush.test.mjs index edd6ed0..9596054 100644 --- a/packages/runtime-node/test/autoflush.test.mjs +++ b/packages/runtime-node/test/autoflush.test.mjs @@ -90,11 +90,34 @@ test("e2e: captured exception attributes are redacted on the wire", async () => assert.ok(!wire.includes("jane.doe@example.com"), "raw email leaked"); assert.ok(!wire.includes("supersecrettoken123456"), "raw bearer token leaked"); assert.ok(!wire.includes("eyJhbGciOiJIUzI1NiJ9"), "raw JWT leaked"); + assert.ok(!wire.includes("rawcookievalue"), "nested cookie leaked"); + + // Secrets inside exception messages, stacks, and status messages. + for (const secret of [ + "hunter2pg", + "4111 1111 1111 1111", + "4eC39HqLyjWDarjtT1zdp7dc", + "CUST-123456", // customer-supplied additionalValuePatterns + "redispass99", // thrown inside withProcessSpan + "should-not-leak", // LlmCallHandle.setAttributes + "abcdefghijklmnopqrstuvwxyz0123", // provider error echoing an sk-proj key + // Third-party span: only the export-time scrub sees these. + "foreigntoken123", + "foreignbearer12345", + "foreignpw1", + ]) { + assert.ok(!wire.includes(secret), `secret leaked on the wire: ${secret}`); + } // Mask present; non-sensitive context intact. assert.ok(wire.includes("[redacted]"), "expected mask marker"); assert.ok(wire.includes("o-1"), "non-sensitive attribute was dropped"); assert.ok(wire.includes("boom: order failed"), "exception message missing"); + assert.ok(wire.includes("connect ECONNREFUSED postgres://[redacted]@db.internal:5432/app"), + "scrubbed message should keep its non-secret context"); + assert.ok(wire.includes("page=2"), "non-secret query params should survive"); + // Usage counters on third-party LLM spans must survive for cost tracking. + assert.match(wire, /"ai\.usage\.promptTokens","value":\{"intValue":42/); // Debug mode reported exports. assert.match(err, /exported \d+ span/); diff --git a/packages/runtime-node/test/fixtures/e2e-redaction.mjs b/packages/runtime-node/test/fixtures/e2e-redaction.mjs index 8567860..e9ddf21 100644 --- a/packages/runtime-node/test/fixtures/e2e-redaction.mjs +++ b/packages/runtime-node/test/fixtures/e2e-redaction.mjs @@ -1,6 +1,8 @@ // Child fixture: FULL pipeline e2e — initAutterServer against the parent's -// local collector, capture an exception full of PII, then SIGTERM. Auto-flush -// must push the redacted span out before the process dies. +// local collector, capture exceptions full of PII/secrets (in attributes, +// messages, stacks, third-party spans), then SIGTERM. Auto-flush must push +// the redacted spans out before the process dies. +import { trace } from "@opentelemetry/api"; import { initAutterServer } from "../../dist/index.js"; const autter = initAutterServer({ @@ -9,15 +11,57 @@ const autter = initAutterServer({ service: "e2e-redaction", environment: "test", metricIntervalMs: 3_600_000, // keep metric noise out of this test + traceSampleRate: 1, + redactAttributes: { additionalValuePatterns: [/CUST-\d{6}/] }, }); -setTimeout(() => { +// Fake secrets assembled from fragments so repo secret scanners stay quiet. +const STRIPE = ["sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"].join(""); + +setTimeout(async () => { autter.captureException(new Error("boom: order failed"), { "order.id": "o-1", "user.email": "jane.doe@example.com", auth: "Bearer supersecrettoken123456", "context.jwt": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + // (OTel drops object-valued attributes on its own; redaction must + // still never pass the raw cookie through.) + request: { headers: { cookie: "sid=rawcookievalue" } }, }); + + // The classic leak: a secret inside the error MESSAGE (and so the stack). + autter.captureException( + new Error("connect ECONNREFUSED postgres://admin:hunter2pg@db.internal:5432/app"), + ); + autter.captureMessage(`charge failed for card 4111 1111 1111 1111 key ${STRIPE}`, "warning"); + autter.captureException(new Error("lookup failed for CUST-123456")); + + // A process span whose thrown error carries a secret. + await autter + .withProcessSpan("nightly-sync", () => { + throw new Error("redis://:redispass99@cache:6379 refused"); + }) + .catch(() => {}); + + // LLM provider error echoing a key. + await autter + .withLlmCall({ provider: "openai", model: "gpt-x" }, (llm) => { + llm.setAttributes({ "gen_ai.request.api_key": "should-not-leak", note: "ok" }); + throw new Error("401 Incorrect API key provided: sk-proj-abcdefghijklmnopqrstuvwxyz0123"); + }) + .catch(() => {}); + + // Spans written by someone else (an instrumentation or the app itself + // via the global tracer): only export-time scrubbing can catch these. + const foreign = trace.getTracer("third-party").startSpan("GET /callback"); + foreign.setAttributes({ + "http.url": "https://app.example.test/callback?code=abc&access_token=foreigntoken123&page=2", + "http.request.header.authorization": ["Bearer foreignbearer12345"], + "ai.usage.promptTokens": 42, + }); + foreign.recordException(new Error("upstream said password=foreignpw1 for x")); + foreign.end(); + setTimeout(() => { process.kill(process.pid, "SIGTERM"); }, 100); diff --git a/packages/runtime-node/test/redact.test.mjs b/packages/runtime-node/test/redact.test.mjs index bad04da..53f3827 100644 --- a/packages/runtime-node/test/redact.test.mjs +++ b/packages/runtime-node/test/redact.test.mjs @@ -281,3 +281,84 @@ test("handles circular references without leaking sensitive values", () => { assert.equal(out.context.nested.safe, "ok"); assert.equal(out.context.self, MASK); }); + +// --------------------------------------------------------------------------- +// Shared vectors (test-vectors/redaction.json) — the same cases run against +// the browser SDK, the ingester, and the Python adapter, so every layer +// agrees on what a secret looks like. +// --------------------------------------------------------------------------- +import { readFileSync } from "node:fs"; +import { redactText, sanitizeBrowserPayload } from "../dist/index.js"; + +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +); +const join = (value) => (Array.isArray(value) ? value.join("") : value); + +for (const vector of vectors.text) { + if (vector.skip?.includes("node")) continue; + test(`redactText vector: ${vector.name}`, () => { + assert.equal(redactText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive keys are masked case-insensitively at any depth", () => { + for (const key of vectors.keys.sensitive) { + const out = redactAttributes({ outer: { inner: [{ [key]: "raw-value" }] } }); + assert.equal(out.outer.inner[0][key], MASK, key); + } + for (const key of vectors.keys.safe) { + const out = redactAttributes({ outer: { [key]: "kept" } }); + assert.equal(out.outer[key], "kept", key); + } +}); + +test("secrets inside nested string values are scrubbed, not just keys", () => { + const out = redactAttributes({ + job: { args: ["--db", "postgres://u:pw123@db/x"], note: "Cookie: a=b" }, + }); + assert.deepEqual(out.job.args, ["--db", `postgres://${MASK}@db/x`]); + assert.equal(out.job.note, `Cookie: ${MASK}`); +}); + +test("stringified stack traces are scanned line by line", () => { + const err = new Error("auth failed: Authorization: Bearer abcdefghijklmnop123456"); + const out = redactText(err.stack); + assert.ok(!out.includes("abcdefghijklmnop123456")); + assert.match(out, /^Error: auth failed: Authorization: \[redacted\]\n\s+at /); +}); + +test("custom value patterns apply to every match even without the g flag", () => { + const out = redactText("CUST-1 and CUST-2", { + additionalValuePatterns: [/CUST-\d/], + }); + assert.equal(out, `${MASK} and ${MASK}`); +}); + +test("custom key patterns extend the built-in list", () => { + const out = redactAttributes({ tenant_ref: "t-1", other: "x" }, { + additionalKeyPatterns: [/tenant_ref/], + }); + assert.deepEqual(out, { tenant_ref: MASK, other: "x" }); +}); + +test("the browser relay scrubs message, stack, name and nested context", () => { + const payload = sanitizeBrowserPayload({ + version: 1, + service: "web", + environment: "production", + events: [{ + type: "exception", + timestamp: "2026-01-01T00:00:00.000Z", + message: "fetch failed for https://api.test/x?token=abc123secret", + stack: "Error: x\n at f (https://app.test/a.js:1:1) jane@example.com", + name: "/verify/eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + context: { form: { password: "p@ss", plan: "pro" }, note: "sk-abcdefghijklmnopqrstuvwxyz" }, + }], + }); + const [event] = payload.events; + assert.equal(event.message, `fetch failed for https://api.test/x?token=${MASK}`); + assert.ok(!event.stack.includes("jane@example.com")); + assert.equal(event.name, `/verify/${MASK}`); + assert.deepEqual(event.context, { form: { password: MASK, plan: "pro" }, note: MASK }); +}); diff --git a/test-vectors/redaction.json b/test-vectors/redaction.json new file mode 100644 index 0000000..8bdd62a --- /dev/null +++ b/test-vectors/redaction.json @@ -0,0 +1,196 @@ +{ + "$comment": "Shared secret/PII scrubbing vectors. Every redactor (runtime-node redactText, runtime-browser scrubText, the relay, the otlp-ingester scrubText, and adapters/python redact_text) must map `input` to `expect`. Strings given as arrays are joined with no separator: realistic-looking vendor tokens are split into fragments so secret scanners (GitHub push protection) don't flag the fixture. `skip` lists implementations that deliberately don't cover a case.", + "text": [ + { + "name": "JWT inside an error message", + "input": "invalid session eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c for user", + "expect": "invalid session [redacted] for user" + }, + { + "name": "Authorization: Bearer header line", + "input": "request failed: Authorization: Bearer abcdefghijklmnopqrstuvwxyz1234567890", + "expect": "request failed: Authorization: [redacted]" + }, + { + "name": "bare bearer token", + "input": "sent Bearer abcdefghijklmnopqrstuvwxyz1234567890 upstream", + "expect": "sent [redacted] upstream" + }, + { + "name": "Basic auth credentials", + "input": "auth header Basic dXNlcjpwYXNzd29yZDEyMw== rejected", + "expect": "auth header [redacted] rejected" + }, + { + "name": "'basic' as an ordinary word survives", + "input": "requires basic authentication", + "expect": "requires basic authentication" + }, + { + "name": "Cookie header keeps no cookie values", + "input": "Cookie: sid=abc123; theme=dark", + "expect": "Cookie: [redacted]" + }, + { + "name": "Set-Cookie header", + "input": "Set-Cookie: session=s3cr3tvalue; Path=/; HttpOnly", + "expect": "Set-Cookie: [redacted]" + }, + { + "name": "headers serialised as JSON", + "input": "{\"authorization\":\"Bearer abcdefghijklmnopqrstuvwxyz\",\"cookie\":\"sid=abc; x=y\",\"host\":\"api.internal\"}", + "expect": "{\"authorization\":\"[redacted]\",\"cookie\":\"[redacted]\",\"host\":\"api.internal\"}" + }, + { + "name": "postgres connection string in ECONNREFUSED message", + "input": "connect ECONNREFUSED postgres://admin:hunter2@db.internal:5432/app", + "expect": "connect ECONNREFUSED postgres://[redacted]@db.internal:5432/app" + }, + { + "name": "mongodb+srv connection string", + "input": "MongoServerError at mongodb+srv://svc:Pa55word@cluster0.abcde.mongodb.net/prod", + "expect": "MongoServerError at mongodb+srv://[redacted]@cluster0.abcde.mongodb.net/prod" + }, + { + "name": "redis URL with empty username", + "input": "redis://:hunter2@cache.internal:6379/0 unreachable", + "expect": "redis://[redacted]@cache.internal:6379/0 unreachable" + }, + { + "name": "mysql URL whose password contains @", + "input": "mysql://root:p@ss@db:3306/shop", + "expect": "mysql://[redacted]@db:3306/shop" + }, + { + "name": "AWS access key id", + "input": "using AKIAIOSFODNN7EXAMPLE for s3", + "expect": "using [redacted] for s3" + }, + { + "name": "OpenAI project key (sk-proj-)", + "input": ["key ", "sk-proj-", "abcdefghijklmnopqrstuvwxyz0123456789", " rejected"], + "expect": "key [redacted] rejected" + }, + { + "name": "Anthropic key (sk-ant-)", + "input": ["x-api-key ", "sk-ant-api03-", "abcdefghijklmnopqrstuvwxyz0123"], + "expect": "x-api-key [redacted]" + }, + { + "name": "Stripe live secret key", + "input": ["stripe ", "sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc", " failed"], + "expect": "stripe [redacted] failed" + }, + { + "name": "GitHub classic and fine-grained tokens", + "skip": ["browser"], + "input": ["ghp_", "abcdefghijklmnopqrstuvwxyz1234567890", " and ", "github_pat_", "11ABCDEFG0abcdefghijklmnopqrstuvwxyz"], + "expect": "[redacted] and [redacted]" + }, + { + "name": "Slack bot token", + "input": ["xox", "b-123456789012-abcdefghijklmnopqrstuv", " end"], + "expect": "[redacted] end" + }, + { + "name": "Google API key (AIza)", + "input": ["maps ", "AIza", "SyA1234567890abcdefghijklmnopqrstu", " quota"], + "expect": "maps [redacted] quota" + }, + { + "name": "PEM private key block", + "skip": ["browser"], + "input": "cfg -----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAabc\n-----END RSA PRIVATE KEY----- loaded", + "expect": "cfg [redacted] loaded" + }, + { + "name": "token and password in a URL query string", + "input": "GET https://api.example.test/cb?token=abc123def&page=2&password=hunter2", + "expect": "GET https://api.example.test/cb?token=[redacted]&page=2&password=[redacted]" + }, + { + "name": "api_key and access_token query params", + "input": "/v1/data?api_key=k-12345&access_token=tok_98765&x=1", + "expect": "/v1/data?api_key=[redacted]&access_token=[redacted]&x=1" + }, + { + "name": "password assignment in free text and JSON", + "input": "login failed password=hunter2 body={\"password\":\"hunter 2\",\"user\":\"bob\"}", + "expect": "login failed password=[redacted] body={\"password\":\"[redacted]\",\"user\":\"bob\"}" + }, + { + "name": "client_secret and DB_PASSWORD env style", + "input": "client_secret: abcd1234 DB_PASSWORD=s3cret!", + "expect": "client_secret: [redacted] DB_PASSWORD=[redacted]" + }, + { + "name": "JSON parse error message is left alone", + "input": "Unexpected token '<', \"