From 7d936c61b2f6f744c248421028f4b05b48db6935 Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Wed, 7 Oct 2026 23:49:31 +0530 Subject: [PATCH] Scrub secrets from error messages, stacks, and every outbound field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exception data is what reaches LLM-generated fixes, and it routinely carries secrets: `connect ECONNREFUSED postgres://admin:hunter2@…`, provider errors echoing API keys, `?token=` in request URLs, cookies and bearer tokens in serialized headers. Redaction previously covered only custom attributes (and only top-level, email-only, in the browser), so messages, stack traces, span status messages, third-party spans, relayed browser events and anything an old SDK or plain OTel sender shipped went to storage and the sink verbatim. SDKs (scrub before anything leaves the customer's process, on by default): - runtime-node: scrub exception messages/stacks/status messages in captureException/captureMessage (incl. pre-init fallbacks), process spans, withLlmCall/trackLlmCall errors, LlmCallHandle.setAttributes and instrumentLlmClient; add a RedactingSpanExporter so every exported span (HTTP URLs, headersToSpanAttributes, third-party instrumentations, app-level recordException) is scrubbed; relay scrubs browser message/stack/name/nested context (new `redact` option); new redactText - runtime-browser: scrub messages, stacks, names and nested context; new `redact: { keys, values } | false` option and scrubText export (3.43 kB -> 4.14 kB brotlied, budget 5 kB) - python adapter: new stdlib-only redact.py; caught-exception sampler scrubs tracebacks with it Patterns: add Basic auth, Authorization/Cookie/Set-Cookie header text, connection strings with empty user or `@` in the password, sk-proj/ sk-ant, Stripe, GitHub fine-grained, GitLab, npm, SendGrid, Google keys, password=/?token=/?api_key= assignments, Luhn-valid cards; keys for any *authorization header, session/sessionid/sid, pwd, dsn, bounded ssn. Custom value patterns now apply globally. Email and URL-scheme quantifiers are bounded (email was quadratic on long strings: ~1 s per 32 KB, now ~10 ms). Ingester (second line of defence for old SDKs / third-party OTLP senders, before ClickHouse and the sink that feeds LLM fixing): scrub occurrence message/stack/route/errorType, span names, deep browser context and LLM-call attributes with the shared patterns; extra patterns via AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS. Parity: test-vectors/redaction.json runs against node, browser, ingester and Python; CI now runs the Python adapter tests. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 16 ++ adapters/python/caught_exceptions.py | 13 +- adapters/python/redact.py | 198 ++++++++++++++ adapters/python/test_redact.py | 111 ++++++++ docs/ARCHITECTURE.md | 19 +- docs/CONTINUOUS-DETECTION.md | 8 + packages/otlp-ingester/README.md | 2 + packages/otlp-ingester/src/config.ts | 24 ++ packages/otlp-ingester/src/context.ts | 25 +- packages/otlp-ingester/src/llm.ts | 24 +- .../otlp-ingester/src/normalize-browser.ts | 32 +-- packages/otlp-ingester/src/normalize-otlp.ts | 5 +- packages/otlp-ingester/src/redact.test.ts | 209 +++++++++++++++ packages/otlp-ingester/src/redact.ts | 119 +++++++++ packages/otlp-ingester/src/server.ts | 10 +- packages/runtime-browser/README.md | 28 +- packages/runtime-browser/src/index.ts | 90 +++++-- .../runtime-browser/test/redact-e2e.test.mjs | 50 ++++ .../test/redact-vectors.test.mjs | 38 +++ packages/runtime-next/src/server.ts | 1 + packages/runtime-node/README.md | 50 ++-- packages/runtime-node/src/index.ts | 1 + packages/runtime-node/src/llm-instrument.ts | 12 +- packages/runtime-node/src/redact-exporter.ts | 81 ++++++ packages/runtime-node/src/redact.ts | 242 +++++++++++++----- packages/runtime-node/src/relay.ts | 46 +++- packages/runtime-node/src/server.ts | 170 ++++++------ packages/runtime-node/test/autoflush.test.mjs | 23 ++ .../test/fixtures/e2e-redaction.mjs | 50 +++- packages/runtime-node/test/redact.test.mjs | 81 ++++++ test-vectors/redaction.json | 196 ++++++++++++++ 32 files changed, 1736 insertions(+), 239 deletions(-) create mode 100644 adapters/python/redact.py create mode 100644 adapters/python/test_redact.py create mode 100644 packages/otlp-ingester/src/redact.test.ts create mode 100644 packages/otlp-ingester/src/redact.ts create mode 100644 packages/runtime-browser/test/redact-e2e.test.mjs create mode 100644 packages/runtime-browser/test/redact-vectors.test.mjs create mode 100644 packages/runtime-node/src/redact-exporter.ts create mode 100644 test-vectors/redaction.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7928ece..140c957 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,3 +19,4 @@ jobs: - run: npm run test -w @autter/runtime-browser - run: npm run test -w @autter/otlp-ingester - run: npm run size -w @autter/runtime-browser + - run: python3 -m unittest discover -s adapters/python -v diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c0cd1b..4b54180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [Unreleased] + +### Security + +- Scrub secrets and PII from exception messages, stack traces and span status messages, not only custom attributes; re-scrub every span at export so third-party instrumentations, HTTP URLs (`?token=`) and `recordException` calls are covered; scrub `LlmCallHandle.setAttributes`, `instrumentLlmClient` attributes and provider errors, and module-level captures made before `initAutterServer` (`runtime-node`). +- New value patterns everywhere: Basic auth, `Authorization:`/`Cookie:`/`Set-Cookie:` header text, connection strings with empty usernames or `@` in the password, `sk-proj-`/`sk-ant-`, Stripe, GitHub fine-grained, GitLab, npm, SendGrid and Google API keys, `password=`/`?token=`/`?api_key=` assignments, Luhn-valid card numbers. New sensitive keys: any `…authorization` header key, `session`/`sessionid`/`sid`, `pwd`, `dsn`, bounded `ssn`. Custom value patterns now apply globally (`runtime-node`, `runtime-browser`, `otlp-ingester`). +- The browser relay scrubs message, stack, name and nested context (`redact` option) (`runtime-node`). +- Browser SDK scrubs messages, stacks and nested context; new `redact: { keys, values } | false` option and `scrubText` export (`runtime-browser`). +- Ingester scrubs occurrence message/stack/route, span names, browser context (deep) and LLM-call attributes before storage and the sink webhook; extra patterns via `AUTTER_REDACT_VALUE_PATTERNS` / `AUTTER_REDACT_KEY_PATTERNS` (`otlp-ingester`). +- Python adapter: new stdlib-only `adapters/python/redact.py` (`redact_text`, `redact_attributes`, `configure`); the caught-exception sampler scrubs tracebacks with it. +- Shared parity vectors in `test-vectors/redaction.json` run against every implementation. + +### Behavior notes + +- Error messages that contained secrets now fingerprint on their scrubbed form, so such issues may regroup once (they previously split per secret value). + ## [1.3.1] - 2026-09-08 ### Fixes diff --git a/adapters/python/caught_exceptions.py b/adapters/python/caught_exceptions.py index 86de6df..1b2b481 100644 --- a/adapters/python/caught_exceptions.py +++ b/adapters/python/caught_exceptions.py @@ -4,18 +4,26 @@ ones. It has substantial overhead and must be explicitly enabled. Pass an ``emit(name, stack)`` callback that records a standard OTel exception event with ``autter.handled=true`` and ``autter.sampled=true`` on an always-on span. + +The formatted traceback includes ``str(exc)``, so it is scrubbed for secrets +and PII (connection strings, tokens, API keys, emails, …) with +:func:`redact.redact_text` before ``emit`` sees it. """ from __future__ import annotations import random -import re import sys import threading import time import traceback from collections.abc import Callable +try: # imported as part of a package + from .redact import redact_text +except ImportError: # copied next to the app as a plain module + from redact import redact_text + def install_caught_sampler( emit: Callable[[str, str], None], *, sample_rate: float = 0.01, max_per_minute: int = 10 @@ -43,8 +51,7 @@ def hook(frame, event, arg): return hook seen.add(identity) sent += 1 - stack = "".join(traceback.format_exception(exc_type, exc_value, tb, limit=8))[:8000] - stack = re.sub(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", "[redacted]", stack, flags=re.I) + stack = redact_text("".join(traceback.format_exception(exc_type, exc_value, tb, limit=8))[:8000]) try: emit(exc_type.__name__, stack) except Exception: diff --git a/adapters/python/redact.py b/adapters/python/redact.py new file mode 100644 index 0000000..2b45cb9 --- /dev/null +++ b/adapters/python/redact.py @@ -0,0 +1,198 @@ +"""Secret/PII scrubbing for Python services reporting to Autter Runtime. + +Python exceptions routinely carry secrets in their *message* (``str(exc)``) +and therefore in every formatted traceback: ``psycopg2.OperationalError: +connection to postgres://admin:hunter2@db failed``, ``401 Incorrect API key +provided: sk-...``. Scrub before anything leaves the process: + + from redact import redact_text, redact_attributes + + span.add_event("exception", { + "exception.type": type(exc).__name__, + "exception.message": redact_text(str(exc)), + "exception.stacktrace": redact_text("".join(traceback.format_exception(exc))), + }) + span.set_attributes(redact_attributes(custom_context)) + +Same patterns as ``redactText``/``redactAttributes`` in @autter/runtime-node; +parity is enforced by the shared ``test-vectors/redaction.json``. Secure by +default; extend with :func:`configure` (your own value/key patterns). The +Autter ingester scrubs again server-side as a second line of defence. + +Stdlib only. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +MASK = "[redacted]" + +_PRIVATE_KEY_BLOCK = re.compile( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)" +) +_PREFIXED_SECRET = re.compile( + r"\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}" + r"|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}" + r"|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}" + r"|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}" + r"|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})" +) +_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}") +_AUTH_SCHEME = re.compile( + r"\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}" + r"|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})", + re.I, +) +_HEADER_VALUE = re.compile( + r"((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)[\"']?\s*[:=]\s*[\"']?)[^\"'\r\n]+", + re.I | re.M, +) +_SECRET_ASSIGNMENT = re.compile( + r"(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key" + r"|credential|signature|session[_-]?id|sessionid|ssn)" + r"([\"']?\s*[:=]\s*)" + r"(\"[^\"\r\n]*\"|'[^'\r\n]*'|[^\s\"'&,;)}\]\[<>]+)", + re.I, +) +_URL_CREDENTIALS = re.compile( + r"\b([a-z][a-z0-9+.-]{0,31}://)(?:[^\s/:@\"'<>]*:[^\s/\"'<>]*|[^\s/:@\"'<>]{16,})@", re.I +) +_CARD = re.compile( + r"\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b" +) +_EMAIL = re.compile(r"[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}", re.I) + +_SENSITIVE_KEYS = [ + re.compile(p) + for p in ( + r"e-?mail", + r"pass(word|wd|phrase)|^pass$|(^|[._-])pwd$", + r"token", + r"secret", + r"credential", + r"(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key", + r"authori[sz]ation|^auth(-|_|$)|bearer", + r"cookie", + r"(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid", + r"phone|msisdn", + r"(^|[^a-z])ssn($|[^a-z])|social[-_ ]?security", + r"cvv|cvc|card([-_. ]?(number|num|no))?$", + r"credit[-_.]?card", + r"connection[-_.]?string|(^|[._-])dsn$", + r"recovery[-_.]?code|\botp\b|magic[-_.]?link", + ) +] +# Numeric GenAI usage counters keep their value even though "token" matches. +_USAGE_KEY = re.compile(r"(^|\.)(input|output|total|prompt|completion)_?tokens$|token_count$", re.I) + +_extra_values: list[re.Pattern[str]] = [] +_extra_keys: list[re.Pattern[str]] = [] +_enabled = True + +_MAX_DEPTH = 8 +_MAX_ITEMS = 1000 + + +def configure( + *, + value_patterns: Iterable[str | re.Pattern[str]] = (), + key_patterns: Iterable[str | re.Pattern[str]] = (), + enabled: bool = True, +) -> None: + """Add your own patterns on top of the built-ins (replaces earlier extras). + + ``value_patterns`` are masked wherever they occur inside strings; + ``key_patterns`` mask whole values whose key matches (case-insensitive). + ``enabled=False`` turns scrubbing off (not recommended). + """ + global _extra_values, _extra_keys, _enabled + _extra_values = [p if isinstance(p, re.Pattern) else re.compile(p, re.I) for p in value_patterns] + _extra_keys = [p if isinstance(p, re.Pattern) else re.compile(p, re.I) for p in key_patterns] + _enabled = enabled + + +def _luhn_valid(candidate: str) -> bool: + digits = [int(c) for c in candidate if c.isdigit()] + if not 13 <= len(digits) <= 19: + return False + total = 0 + for i, d in enumerate(reversed(digits)): + if i % 2 == 1: + d *= 2 + if d > 9: + d -= 9 + total += d + return total % 10 == 0 + + +def _mask_assignment(match: re.Match[str]) -> str: + key, sep, value = match.group(1), match.group(2), match.group(3) + if value[:1] in ("'", '"'): + return f"{key}{sep}{value[0]}{MASK}{value[0]}" + return f"{key}{sep}{MASK}" + + +def redact_text(value: str) -> str: + """Scrub secrets/PII embedded in one string; the rest survives.""" + if not _enabled or not isinstance(value, str) or not value: + return value + out = _PRIVATE_KEY_BLOCK.sub(MASK, value) + out = _PREFIXED_SECRET.sub(MASK, out) + out = _JWT.sub(MASK, out) + out = _AUTH_SCHEME.sub(MASK, out) + out = _HEADER_VALUE.sub(lambda m: m.group(1) + MASK, out) + out = _SECRET_ASSIGNMENT.sub(_mask_assignment, out) + out = _URL_CREDENTIALS.sub(lambda m: m.group(1) + MASK + "@", out) + out = _CARD.sub(lambda m: MASK if _luhn_valid(m.group(0)) else m.group(0), out) + out = _EMAIL.sub(MASK, out) + for pattern in _extra_values: + out = pattern.sub(MASK, out) + return out + + +def is_sensitive_key(key: str) -> bool: + lowered = key.lower() + return any(p.search(lowered) for p in _SENSITIVE_KEYS) or any(p.search(key) for p in _extra_keys) + + +def _redact_value(key: str, value: Any, depth: int) -> Any: + if isinstance(value, bool) or value is None: + return value + if isinstance(value, (int, float)): + if _USAGE_KEY.search(key) and value >= 0: + return value + return MASK if is_sensitive_key(key) else value + if is_sensitive_key(key): + return MASK + if isinstance(value, str): + return redact_text(value) + if depth >= _MAX_DEPTH: + return MASK + if isinstance(value, Mapping): + return _redact_mapping(value, depth + 1) + if isinstance(value, (list, tuple)): + return [_redact_value("", item, depth + 1) for item in list(value)[:_MAX_ITEMS]] + return redact_text(str(value)) + + +def _redact_mapping(mapping: Mapping[Any, Any], depth: int) -> dict[str, Any]: + out: dict[str, Any] = {} + for i, (key, value) in enumerate(mapping.items()): + if i >= _MAX_ITEMS: + out["__redaction_truncated__"] = MASK + break + out[str(key)] = _redact_value(str(key), value, depth) + return out + + +def redact_attributes(attributes: Mapping[str, Any] | None) -> dict[str, Any]: + """Copy of ``attributes`` with sensitive keys masked (any depth) and + secrets scrubbed inside string values. Never mutates the input.""" + if not attributes: + return {} + if not _enabled: + return dict(attributes) + return _redact_mapping(attributes, 0) diff --git a/adapters/python/test_redact.py b/adapters/python/test_redact.py new file mode 100644 index 0000000..2f42694 --- /dev/null +++ b/adapters/python/test_redact.py @@ -0,0 +1,111 @@ +"""Tests for the Python adapter's redaction (stdlib unittest, no deps). + +Run: python3 -m unittest discover -s adapters/python -v +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import redact # noqa: E402 +from caught_exceptions import install_caught_sampler # noqa: E402 +from redact import MASK, configure, redact_attributes, redact_text # noqa: E402 + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "test-vectors", "redaction.json") +with open(VECTORS_PATH, encoding="utf-8") as fh: + VECTORS = json.load(fh) + + +def _join(value): + return "".join(value) if isinstance(value, list) else value + + +class SharedVectors(unittest.TestCase): + def test_text_vectors(self): + for vector in VECTORS["text"]: + if "python" in vector.get("skip", []): + continue + with self.subTest(vector["name"]): + self.assertEqual(redact_text(_join(vector["input"])), _join(vector["expect"])) + + def test_sensitive_keys_masked_at_any_depth(self): + for key in VECTORS["keys"]["sensitive"]: + with self.subTest(key): + out = redact_attributes({"outer": {"items": [{key: "raw"}]}}) + self.assertEqual(out["outer"]["items"][0][key], MASK) + + def test_safe_keys_survive(self): + for key in VECTORS["keys"]["safe"]: + with self.subTest(key): + self.assertEqual(redact_attributes({"outer": {key: "kept"}})["outer"][key], "kept") + + +class Attributes(unittest.TestCase): + def test_nested_values_and_usage_counters(self): + out = redact_attributes( + { + "db": {"dsn": "postgres://u:p@h/db", "args": ["redis://:pw@cache:6379"]}, + "gen_ai.usage.input_tokens": 12, + "max_tokens": 100, + "note": "contact jane@example.com", + } + ) + self.assertEqual(out["db"]["dsn"], MASK) + self.assertEqual(out["db"]["args"], [f"redis://{MASK}@cache:6379"]) + self.assertEqual(out["gen_ai.usage.input_tokens"], 12) + self.assertEqual(out["max_tokens"], MASK) + self.assertEqual(out["note"], f"contact {MASK}") + + def test_does_not_mutate_input(self): + original = {"password": "x", "nested": {"token": "y"}} + redact_attributes(original) + self.assertEqual(original, {"password": "x", "nested": {"token": "y"}}) + + def test_custom_patterns(self): + configure(value_patterns=[r"CUST-\d{4}"], key_patterns=[r"^tenant_ref$"]) + try: + self.assertEqual(redact_text("CUST-1234 / CUST-9999"), f"{MASK} / {MASK}") + self.assertEqual(redact_attributes({"tenant_ref": "t"}), {"tenant_ref": MASK}) + finally: + configure() + + def test_opt_out(self): + configure(enabled=False) + try: + self.assertEqual(redact_text("password=x"), "password=x") + finally: + configure() + self.assertTrue(redact._enabled) + + +class CaughtExceptionSampler(unittest.TestCase): + def test_traceback_message_is_scrubbed_before_emit(self): + emitted: list[tuple[str, str]] = [] + + def connect(): + raise ConnectionError("could not connect to postgres://admin:hunter2@db.internal:5432/app") + + stop = install_caught_sampler(lambda name, stack: emitted.append((name, stack)), sample_rate=1.0) + try: + try: + connect() + except ConnectionError: + pass + finally: + stop() + + self.assertTrue(emitted, "sampler emitted nothing") + name, stack = emitted[0] + self.assertEqual(name, "ConnectionError") + self.assertNotIn("hunter2", stack) + self.assertIn(f"postgres://{MASK}@db.internal:5432/app", stack) + self.assertIn("in connect", stack) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c038fb7..1ad0ed1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -222,11 +222,20 @@ counters, not an analytics event store). Forbidden at the schema level (rejected/stripped): full URLs with query strings, cookies, DOM content, form values, request headers/bodies, emails. -Server-side custom attributes are guarded at the source instead: the Node -SDK masks email/token/credential-shaped values and sensitive-keyed -attributes before export (`redactAttributes`, on by default), so OTLP spans -never carry a stray `user.email` even though the OTLP schema itself accepts -free-form attributes. +Server-side telemetry is guarded at the source instead: the Node SDK masks +secret/PII-shaped values (tokens, keys, credentials in connection strings +and URLs, emails, card numbers) and sensitive-keyed attributes in +attributes, exception messages, stack traces and status messages, and +re-scrubs every span at export (`redactAttributes`, on by default). The +browser SDK, the relay and the Python adapter apply the same patterns. + +Because old SDKs and plain OTel senders scrub nothing, the ingester scrubs +again before storage and before the sink webhook (which feeds downstream +LLM-assisted fixing): occurrence message/stack/route, span names, all +attribute bags, and LLM-call attributes (`src/redact.ts`; extend with +`AUTTER_REDACT_VALUE_PATTERNS` / `AUTTER_REDACT_KEY_PATTERNS`, JSON arrays +of regex sources). Messages containing secrets therefore fingerprint on +their scrubbed form. ## Sink webhook (v1) diff --git a/docs/CONTINUOUS-DETECTION.md b/docs/CONTINUOUS-DETECTION.md index 5410629..823e5eb 100644 --- a/docs/CONTINUOUS-DETECTION.md +++ b/docs/CONTINUOUS-DETECTION.md @@ -67,6 +67,14 @@ OTel `exception` event with `autter.handled=true` and overhead, applies to the current and newly created threads, and should be enabled only for targeted diagnosis. Other languages can send the same event. +The sampler scrubs each traceback (which includes `str(exc)`) for secrets +and PII before calling `emit`, using `adapters/python/redact.py` (stdlib +only; copy it next to `caught_exceptions.py`). Use the same module for your +own `record_exception` calls — `redact_text(str(exc))`, +`redact_text(traceback_text)`, `redact_attributes(context)` — and +`configure(value_patterns=[...], key_patterns=[...])` for custom patterns. +The ingester scrubs again server-side. + ```python from opentelemetry import trace from caught_exceptions import install_caught_sampler diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 1f3a360..a993628 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -66,6 +66,8 @@ The validator webhook may return the same extra fields: | `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) | | `OCCURRENCE_TTL_DAYS` / `SPAN_TTL_DAYS` / `METRICS_TTL_DAYS` | `14` / `7` / `90` | ClickHouse TTLs (applied at table creation) | | `LLM_CALL_TTL_DAYS` | `90` | Retention for `runtime_llm_calls` rows | +| `AUTTER_REDACT_VALUE_PATTERNS` | — | JSON array of extra regex sources masked in stored/forwarded text and attribute values (built-in secret/PII patterns always apply) | +| `AUTTER_REDACT_KEY_PATTERNS` | — | JSON array of extra regex sources for attribute keys whose values are masked wholesale | ## LLM / GenAI calls diff --git a/packages/otlp-ingester/src/config.ts b/packages/otlp-ingester/src/config.ts index 45d1d5c..e64679f 100644 --- a/packages/otlp-ingester/src/config.ts +++ b/packages/otlp-ingester/src/config.ts @@ -38,6 +38,15 @@ export interface IngesterConfig { spanTtlDays: number; metricsTtlDays: number; llmCallTtlDays: number; + /** + * Extra regex sources scrubbed from stored/forwarded free text and + * attribute values, and extra regex sources for sensitive attribute + * keys — on top of the built-in secret/PII patterns (see redact.ts). + * Env: AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS (JSON + * arrays of strings). + */ + redactValuePatterns?: string[]; + redactKeyPatterns?: string[]; } function intEnv(name: string, fallback: number): number { @@ -47,6 +56,19 @@ function intEnv(name: string, fallback: number): number { return Number.isFinite(value) && value > 0 ? value : fallback; } +/** JSON array of regex sources; invalid input fails startup loudly rather + * than silently running without the operator's extra patterns. */ +function patternListEnv(name: string): string[] { + const raw = process.env[name]; + if (!raw) return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed) || !parsed.every((p) => typeof p === "string")) { + throw new Error(`${name} must be a JSON array of regex source strings`); + } + for (const source of parsed) new RegExp(source as string); + return parsed as string[]; +} + function parseIngestKeys(raw: string | undefined): StaticIngestKey[] { if (!raw) return []; try { @@ -91,6 +113,8 @@ export function loadConfig(): IngesterConfig { // LLM calls keep the metrics horizon, not the span one — cost trends // need months, and per-call volume is small next to HTTP spans. llmCallTtlDays: intEnv("LLM_CALL_TTL_DAYS", 90), + redactValuePatterns: patternListEnv("AUTTER_REDACT_VALUE_PATTERNS"), + redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"), }; if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) { throw new Error( diff --git a/packages/otlp-ingester/src/context.ts b/packages/otlp-ingester/src/context.ts index ff5e30d..df80d9e 100644 --- a/packages/otlp-ingester/src/context.ts +++ b/packages/otlp-ingester/src/context.ts @@ -1,22 +1,15 @@ +import { isSensitiveKey, scrubText } from "./redact.js"; + /** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */ export function sanitizeRuntimeContext( input: unknown, ): Record { let budget = 512; const seen = new WeakSet(); + // Shared secret/PII patterns (redact.ts), then drop URL query strings + // and fragments outright — whitelist-style privacy for free text. const scrub = (value: string) => - value - .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted]") - .replace( - /\b(?:bearer\s+[A-Za-z0-9._~+\/=~-]{10,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|(?:AKIA|ASIA)[A-Z0-9]{16})\b/gi, - "[redacted]", - ) - .replace( - /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g, - "[redacted]", - ) - .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi, "$1[redacted]@") - .replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); + scrubText(value).replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); const visit = (value: unknown, key: string, depth: number): unknown => { if (--budget < 0 || depth > 6) return "[truncated]"; if (/__proto__|constructor|prototype/i.test(key)) return undefined; @@ -25,13 +18,7 @@ export function sanitizeRuntimeContext( typeof value === "number" && Number.isFinite(value) && value >= 0; - if ( - !usage && - /password|passwd|secret|token|credential|authorization|cookie|email|phone|ssn|card[._-]?number|connection[._-]?string|api[._-]?key|private[._-]?key|request[._-]?body|response[._-]?body|headers|url[._-]?query/i.test( - key, - ) - ) - return "[redacted]"; + if (!usage && key && isSensitiveKey(key)) return "[redacted]"; if (typeof value === "string") { // Serialized custom context crosses the same privacy boundary as nested values. if (/^\s*[\[{]/.test(value)) { diff --git a/packages/otlp-ingester/src/llm.ts b/packages/otlp-ingester/src/llm.ts index a2ad074..a797c3a 100644 --- a/packages/otlp-ingester/src/llm.ts +++ b/packages/otlp-ingester/src/llm.ts @@ -1,4 +1,5 @@ import { estimateLlmCostUsd } from "./llm-pricing.js"; +import { isSensitiveKey, scrubText } from "./redact.js"; import type { RuntimeLlmCall } from "./types.js"; /** @@ -160,7 +161,7 @@ export function extractLlmCall( 200, ) : "", - userId: ( + userId: scrubText( strAttr( attrs, "autter.user_id", @@ -168,21 +169,27 @@ export function extractLlmCall( "ai.telemetry.metadata.user_id", "enduser.id", "user.id", - ) ?? "" + ) ?? "", ).slice(0, 200), - sessionId: ( + sessionId: scrubText( strAttr( attrs, "autter.session_id", "ai.telemetry.metadata.sessionId", "session.id", - ) ?? "" + ) ?? "", ).slice(0, 200), attributes: llmAttributeSubset(attrs), startedAt: facts.startedAt, }; } +/** Numeric token/usage counters are kept even though their keys contain + * "token" — cost tracking depends on them. */ +function isUsageCount(key: string, value: string): boolean { + return /tokens?$|token_?count$|usage/i.test(key) && /^\d+(\.\d+)?$/.test(value); +} + /** * Drill-down context stored as JSON alongside the typed columns: only the * LLM-relevant attribute families, size-capped so a prompt accidentally @@ -201,7 +208,14 @@ function llmAttributeSubset( ) { continue; } - const trimmed = value.length > 500 ? `${value.slice(0, 500)}…` : value; + // Prompts/completions (ai.prompt, gen_ai.prompt, …) land here: scrub + // before the size cap so a secret straddling it can't half-survive. + const safe = isUsageCount(key, value) + ? value + : isSensitiveKey(key) + ? "[redacted]" + : scrubText(value); + const trimmed = safe.length > 500 ? `${safe.slice(0, 500)}…` : safe; size += key.length + trimmed.length; if (size > 4000) break; subset[key] = trimmed; diff --git a/packages/otlp-ingester/src/normalize-browser.ts b/packages/otlp-ingester/src/normalize-browser.ts index 847d58e..a6e5d92 100644 --- a/packages/otlp-ingester/src/normalize-browser.ts +++ b/packages/otlp-ingester/src/normalize-browser.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { sanitizeRuntimeContext } from "./context.js"; import { normalizeRoute } from "./fingerprint.js"; +import { scrubText } from "./redact.js"; import { asSeverity, type RuntimeMetricPoint, @@ -65,26 +67,19 @@ const TYPE_TO_ERROR_TYPE: Record = { }; // Content-level gate for the free-form `context` bag. The schema whitelist -// above is structural; this masks obvious PII/secrets inside whatever a -// (possibly outdated) SDK still sends: values under sensitive-looking keys -// and email-shaped strings — mirroring redactAttributes() in -// @autter/runtime-node and redactContext() in @autter/runtime-browser. -const SENSITIVE_KEY_RE = - /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +// above is structural; this masks secrets/PII inside whatever a (possibly +// outdated) SDK still sends — sensitive keys and secret-shaped values at any +// depth, via the same sanitiser as OTLP attributes. Message, stack, and route +// are scrubbed centrally before fingerprinting (server.ts fingerprintAll). +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; const REDACTED = "[redacted]"; function scrubContext(context: Record): Record { - const out: Record = {}; - for (const [key, value] of Object.entries(context)) { - if (value === undefined || value === null) continue; - out[key] = SENSITIVE_KEY_RE.test(key) - ? REDACTED - : typeof value === "string" - ? value.replace(EMAIL_VALUE_RE, REDACTED) - : value; - } - return out; + return sanitizeRuntimeContext( + Object.fromEntries( + Object.entries(context).filter(([, value]) => value !== undefined && value !== null), + ), + ); } /** Default severity per event type when the SDK doesn't say. */ @@ -113,6 +108,7 @@ export function normalizeBrowserPayload( occurredAt: Date, counts: { requestCount?: number; errorCount?: number; sessionCount?: number }, ): void { + route = scrubText(route); const bucketAt = new Date( Math.floor(occurredAt.getTime() / 60_000) * 60_000, ); @@ -156,7 +152,7 @@ export function normalizeBrowserPayload( const name = event.name === "browser.longtask" ? "browser.longtask" : `browser.resource:${normalizeRoute((event.name ?? "").replace(/^browser\.resource:/, "").split("?")[0]!.replace(EMAIL_VALUE_RE, REDACTED))}`; bumpRollup(name, occurredAt, { requestCount: 1 }); - const point = rollups.get(`${name} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`); + const point = rollups.get(`${scrubText(name)} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`); if (point) point.durationSumMs += event.durationMs ?? 0; continue; } diff --git a/packages/otlp-ingester/src/normalize-otlp.ts b/packages/otlp-ingester/src/normalize-otlp.ts index e939916..09cb307 100644 --- a/packages/otlp-ingester/src/normalize-otlp.ts +++ b/packages/otlp-ingester/src/normalize-otlp.ts @@ -1,4 +1,5 @@ import { decodeOtlpAttributes, sanitizeRuntimeContext, type OtlpValue } from "./context.js"; +import { scrubText } from "./redact.js"; import { normalizeRoute } from "./fingerprint.js"; import { extractLlmCall } from "./llm.js"; import { @@ -10,7 +11,7 @@ import { type RuntimeSpanRow, } from "./types.js"; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; /** * OTLP/HTTP JSON → runtime signal. Structural types cover only the fields @@ -250,7 +251,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { traceId: span.traceId ?? "", spanId: span.spanId ?? "", parentSpanId: span.parentSpanId ?? null, - name: span.name ?? "unnamed", + name: scrubText(span.name ?? "unnamed").slice(0, 1000), kind, status: isError ? "error" : "ok", route: route ? normalizeRoute(route) : null, diff --git a/packages/otlp-ingester/src/redact.test.ts b/packages/otlp-ingester/src/redact.test.ts new file mode 100644 index 0000000..a25bffb --- /dev/null +++ b/packages/otlp-ingester/src/redact.test.ts @@ -0,0 +1,209 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { after, before, test } from "node:test"; +import type { IngesterConfig } from "./config.js"; +import { sanitizeRuntimeContext } from "./context.js"; +import { configureRedaction, scrubText } from "./redact.js"; +import { createIngesterApp } from "./server.js"; + +/** + * Server-side scrubbing is the backstop for senders that don't scrub (old + * SDKs, plain OTel SDKs in any language). Vectors are shared with the Node + * SDK, browser SDK and Python adapter (test-vectors/redaction.json). + */ + +interface Vector { + name: string; + input: string | string[]; + expect: string | string[]; + skip?: string[]; +} +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +) as { text: Vector[]; keys: { sensitive: string[]; safe: string[] } }; +const join = (value: string | string[]) => (Array.isArray(value) ? value.join("") : value); + +for (const vector of vectors.text) { + if (vector.skip?.includes("ingester")) continue; + test(`scrubText vector: ${vector.name}`, () => { + assert.equal(scrubText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive keys are masked at any depth; safe keys survive", () => { + for (const key of vectors.keys.sensitive) { + const out = sanitizeRuntimeContext({ outer: { list: [{ [key]: "raw" }] } }) as any; + assert.equal(out.outer.list[0][key], "[redacted]", key); + } + for (const key of vectors.keys.safe) { + const out = sanitizeRuntimeContext({ outer: { [key]: "kept" } }) as any; + assert.equal(out.outer[key], "kept", key); + } +}); + +test("operator-supplied patterns extend the built-ins", () => { + configureRedaction({ redactValuePatterns: ["EMP-\\d{4}"], redactKeyPatterns: ["^tenant_ref$"] }); + try { + assert.equal(scrubText("employee EMP-1234 and EMP-9999"), "employee [redacted] and [redacted]"); + assert.deepEqual(sanitizeRuntimeContext({ tenant_ref: "t1", plan: "pro" }), { + tenant_ref: "[redacted]", + plan: "pro", + }); + } finally { + configureRedaction({}); + } +}); + +// --------------------------------------------------------------------------- +// Over the wire: an unscrubbing OTLP sender → ingester → ClickHouse + sink. +// Nothing secret may reach either. +// --------------------------------------------------------------------------- + +let chStub: Server; +let sinkServer: Server; +let appServer: Server; +let ingestUrl: string; +const clickhouseBodies: string[] = []; +const sinkBodies: string[] = []; + +before(async () => { + chStub = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + clickhouseBodies.push(`${req.url}\n${body}`); + res.writeHead(200).end(); + }); + }); + await new Promise((resolve) => chStub.listen(0, resolve)); + sinkServer = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + sinkBodies.push(body); + res.writeHead(202).end(); + }); + }); + await new Promise((resolve) => sinkServer.listen(0, resolve)); + const config: IngesterConfig = { + port: 0, + clickhouseUrl: `http://127.0.0.1:${(chStub.address() as AddressInfo).port}`, + clickhouseUser: "default", + clickhousePassword: "", + clickhouseDatabase: "autter_runtime", + ingestKeys: [{ key: "test-key", orgId: "org-r", repositoryId: "repo-r" }], + keyValidatorUrl: null, + keyValidatorToken: null, + sinkUrl: `http://127.0.0.1:${(sinkServer.address() as AddressInfo).port}/sink`, + sinkToken: null, + sinkMaxAttempts: 3, + sinkMaxBufferedBatches: 10, + sinkMaxBufferedMb: 1, + maxBodyBytes: 1024 * 1024, + rateLimitPerMinute: 300, + clientRateLimitPerMinute: 120, + occurrenceTtlDays: 14, + spanTtlDays: 7, + metricsTtlDays: 90, + llmCallTtlDays: 90, + }; + const { app } = createIngesterApp(config); + appServer = app.listen(0); + await new Promise((resolve) => appServer.once("listening", resolve)); + ingestUrl = `http://127.0.0.1:${(appServer.address() as AddressInfo).port}`; +}); + +after(() => { + appServer.close(); + chStub.close(); + sinkServer.close(); +}); + +const str = (key: string, value: string) => ({ key, value: { stringValue: value } }); +const STRIPE = ["sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"].join(""); + +async function waitFor(check: () => boolean): Promise { + const deadline = Date.now() + 5000; + while (!check()) { + if (Date.now() > deadline) throw new Error("timed out"); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +test("secrets from an unscrubbing OTLP sender never reach ClickHouse or the sink", async () => { + const now = `${BigInt(Date.now()) * 1_000_000n}`; + const res = await fetch(`${ingestUrl}/v1/traces`, { + method: "POST", + headers: { authorization: "Bearer test-key", "content-type": "application/json" }, + body: JSON.stringify({ + resourceSpans: [{ + resource: { attributes: [str("service.name", "py-worker")] }, + scopeSpans: [{ + spans: [ + { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + name: "GET /reset/eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + kind: 2, + startTimeUnixNano: now, + endTimeUnixNano: now, + status: { code: 2, message: "psycopg2: postgres://admin:hunter2pg@db:5432/app" }, + attributes: [ + str("http.url", "https://api.test/cb?access_token=leakytoken99&page=2"), + str("http.request.header.cookie", "sid=rawcookie42"), + ], + events: [{ + name: "exception", + timeUnixNano: now, + attributes: [ + str("exception.type", "OperationalError"), + str("exception.message", `stripe rejected ${STRIPE} for jane@example.com`), + str("exception.stacktrace", 'Traceback (most recent call last):\n File "/app/pay.py", line 9, in charge\n stripe.api_key = "' + STRIPE + '"\nOperationalError: password=hunter3py'), + ], + }], + }, + { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b8", + name: "chat gpt-x", + kind: 3, + startTimeUnixNano: now, + endTimeUnixNano: now, + attributes: [ + str("gen_ai.system", "openai"), + str("gen_ai.request.model", "gpt-x"), + str("gen_ai.prompt", "summarise: my key is sk-proj-abcdefghijklmnopqrstuvwxyz0123"), + { key: "gen_ai.usage.input_tokens", value: { intValue: "321" } }, + str("enduser.id", "jane@example.com"), + ], + }, + ], + }], + }], + }), + }); + assert.equal(res.status, 200); + await waitFor(() => sinkBodies.length > 0); + + const everything = [...clickhouseBodies, ...sinkBodies].join("\n"); + for (const secret of [ + "hunter2pg", + "4eC39HqLyjWDarjtT1zdp7dc", + "jane@example.com", + "hunter3py", + "leakytoken99", + "rawcookie42", + "eyJhbGciOiJIUzI1NiJ9", + "abcdefghijklmnopqrstuvwxyz0123", + ]) { + assert.ok(!everything.includes(secret), `secret reached storage/sink: ${secret}`); + } + // Useful context survives: error type, scrubbed message shape, stack + // frames, and LLM token usage for cost tracking. + assert.ok(everything.includes("OperationalError")); + assert.ok(everything.includes("stripe rejected [redacted] for [redacted]")); + assert.ok(everything.includes('File \\"/app/pay.py\\", line 9, in charge')); + assert.match(everything, /321/); +}); diff --git a/packages/otlp-ingester/src/redact.ts b/packages/otlp-ingester/src/redact.ts new file mode 100644 index 0000000..c3aa926 --- /dev/null +++ b/packages/otlp-ingester/src/redact.ts @@ -0,0 +1,119 @@ +/** + * Server-side secret/PII scrubbing — the ingester's second line of defence. + * + * The SDKs scrub before anything leaves the customer's process, but the + * ingester also accepts older SDK versions and arbitrary third-party OTLP + * senders (any OTel SDK in any language) that scrub nothing. Everything this + * module touches is written to ClickHouse and forwarded to the sink webhook, + * which is where error context is later read for LLM-generated fixes — so + * scrubbing here, before storage, covers every downstream consumer. + * + * Same patterns as redactText() in @autter/runtime-node; parity is enforced + * by the shared test-vectors/redaction.json. + */ + +import type { RuntimeOccurrenceInput } from "./types.js"; + +const MASK = "[redacted]"; + +const PRIVATE_KEY_BLOCK_RE = + /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; +const PREFIXED_SECRET_RE = + /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g; +const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g; +const AUTH_SCHEME_RE = + /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +const HEADER_VALUE_RE = + /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +const SECRET_ASSIGNMENT_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +const URL_CREDENTIALS_RE = + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = + /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g; +const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; + +/** Keys whose values are masked wholesale (tested case-insensitively). */ +export const SENSITIVE_KEY_RE = + /password|passwd|passphrase|(^|[._-])pwd$|secret|token|credential|authori[sz]ation|^auth(-|_|$)|bearer|cookie|e-?mail|phone|(^|[^a-z])ssn($|[^a-z])|cvv|card[._-]?number|connection[._-]?string|(^|[._-])dsn$|(api|access|private|client|signing|encryption)[._-]?key|apikey|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid|request[._-]?body|response[._-]?body|headers|url[._-]?query/i; + +let extraValuePatterns: RegExp[] = []; +let extraKeyPatterns: RegExp[] = []; + +/** + * Deployment-specific patterns on top of the built-ins (env + * AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS, JSON arrays of + * regex sources). Server-side scrubbing has no off switch: it is the + * backstop for senders that don't scrub. + */ +export function configureRedaction(options: { + redactValuePatterns?: string[]; + redactKeyPatterns?: string[]; +}): void { + extraValuePatterns = (options.redactValuePatterns ?? []).map( + (source) => new RegExp(source, "gi"), + ); + extraKeyPatterns = (options.redactKeyPatterns ?? []).map( + (source) => new RegExp(source, "i"), + ); +} + +export function isSensitiveKey(key: string): boolean { + return SENSITIVE_KEY_RE.test(key) || extraKeyPatterns.some((re) => re.test(key)); +} + +function luhnValid(candidate: string): boolean { + const digits = candidate.replace(/\D/g, ""); + if (digits.length < 13 || digits.length > 19) return false; + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = digits.charCodeAt(digits.length - 1 - i) - 48; + if (i % 2 === 1) { + d *= 2; + if (d > 9) d -= 9; + } + sum += d; + } + return sum % 10 === 0; +} + +/** Scrub secrets/PII embedded in one string; the rest of it survives. */ +export function scrubText(value: string): string { + if (typeof value !== "string" || value === "") return value; + let out = value + .replace(PRIVATE_KEY_BLOCK_RE, MASK) + .replace(PREFIXED_SECRET_RE, MASK) + .replace(JWT_RE, MASK) + .replace(AUTH_SCHEME_RE, MASK) + .replace(HEADER_VALUE_RE, (_m, head: string) => head + MASK) + .replace(SECRET_ASSIGNMENT_RE, (_m, key: string, sep: string, val: string) => + key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + MASK + val[0] : MASK), + ) + .replace(URL_CREDENTIALS_RE, (_m, scheme: string) => `${scheme}${MASK}@`) + .replace(CARD_RE, (m) => (luhnValid(m) ? MASK : m)) + .replace(EMAIL_RE, MASK); + for (const re of extraValuePatterns) { + re.lastIndex = 0; + out = out.replace(re, MASK); + } + return out; +} + +/** Nullable convenience for optional columns. */ +export function scrubOptional(value: string | null): string | null { + return value === null ? null : scrubText(value); +} + +/** Scrub the free-text fields of an occurrence (message, stack, route, + * error type). Returns a copy. */ +export function scrubOccurrence( + occurrence: RuntimeOccurrenceInput, +): RuntimeOccurrenceInput { + return { + ...occurrence, + errorType: scrubText(occurrence.errorType), + message: scrubText(occurrence.message), + stack: scrubOptional(occurrence.stack), + route: scrubOptional(occurrence.route), + }; +} diff --git a/packages/otlp-ingester/src/server.ts b/packages/otlp-ingester/src/server.ts index 24d94c4..50e2eea 100644 --- a/packages/otlp-ingester/src/server.ts +++ b/packages/otlp-ingester/src/server.ts @@ -28,6 +28,7 @@ import { decodeProfile } from "./profiles.js"; import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js"; import { validateSourceMap } from "./source-maps.js"; import { SinkForwarder } from "./sink.js"; +import { configureRedaction, scrubOccurrence } from "./redact.js"; import type { IngestContext, RuntimeOccurrence, @@ -42,6 +43,7 @@ export interface IngesterApp { } export function createIngesterApp(config: IngesterConfig): IngesterApp { + configureRedaction(config); const store = new ClickHouseStore(config); // Fingerprinted occurrences feed the consumer's issue grouping, metric // points feed the request/error-rate rollups, LLM calls feed spend @@ -175,7 +177,13 @@ export function createIngesterApp(config: IngesterConfig): IngesterApp { ctx: IngestContext, inputs: RuntimeOccurrenceInput[], ): RuntimeOccurrence[] { - return inputs.map((input, index) => { + return inputs.map((raw, index) => { + // Second line of defence, before fingerprinting, storage, and the + // sink (which feeds LLM fix generation): scrub secrets from the + // free-text fields old SDKs and third-party OTLP senders pass + // through verbatim. Attributes were already sanitised by the + // normalisers (sanitizeRuntimeContext). + const input = scrubOccurrence(raw); const fingerprint = fingerprintOccurrence(input); return { ...input, diff --git a/packages/runtime-browser/README.md b/packages/runtime-browser/README.md index de33b5f..f00676a 100644 --- a/packages/runtime-browser/README.md +++ b/packages/runtime-browser/README.md @@ -82,7 +82,8 @@ initAutterBrowser({ | `setUser(id)` | **Opaque id only** — never an email | | `setContext(ctx)` | Attached to subsequent events | | `flush()` | Force-send the queue (also runs on page hide/unload) | -| `redactContext(ctx)` | Mask obvious PII in a context bag (applied to every event automatically) | +| `redactContext(ctx)` | Mask secrets/PII in a context bag, nested (applied to every event automatically) | +| `scrubText(text)` | Mask secrets/PII inside a string (applied to every message, stack and name automatically) | ## Batching & delivery @@ -101,9 +102,22 @@ Browser and OS are a family plus major version. Filenames and script URLs are query-stripped. For CSP blocks, the directive, blocked resource origin, script path, and a short policy hash are retained. -Custom `context` is free-form, so it is scrubbed before send: values under -sensitive-looking keys (`email`, `password`, `token`, `secret`, `auth`, -`cookie`, `api_key`, `card_number`, …) are replaced with `[redacted]`, and -email-shaped substrings are masked inside ordinary string values. This -mirrors the server SDK's `redactAttributes`; the relay and ingester apply -the same rules as defense-in-depth. +Messages, stack traces and custom `context` (at any depth) are scrubbed +before send: values under sensitive-looking keys (`email`, `password`, +`token`, `secret`, `auth`, `cookie`, `session`, `api_key`, `card_number`, …) +are replaced with `[redacted]`, and secrets inside strings — JWTs, +`Bearer`/`Basic` credentials, `Cookie:`/`Authorization:` text, connection +strings with credentials, common API keys, `?token=`/`password=` +assignments, emails, card numbers — are masked in place. This mirrors the +server SDK's `redactText`/`redactAttributes`; the relay and ingester apply +the full server-side rule set again as defense-in-depth. + +Add your own patterns (or opt out, not recommended): + +```ts +initAutterBrowser({ + ..., + redact: { keys: /^internal_ref$/, values: [/ORD-\d{5}/g] }, + // redact: false, +}); +``` diff --git a/packages/runtime-browser/src/index.ts b/packages/runtime-browser/src/index.ts index 8f84a20..2177ab9 100644 --- a/packages/runtime-browser/src/index.ts +++ b/packages/runtime-browser/src/index.ts @@ -5,8 +5,8 @@ * - zero runtime dependencies, < 5 KB gzipped (CI-enforced) * - no OTel SDK, no console patching, no DOM recording, no offline storage * - privacy by construction: pathname-only routes, no cookies / form values / - * request bodies; query strings stripped everywhere; custom context is - * scrubbed for obvious PII (emails, sensitive keys) before send + * request bodies; query strings stripped everywhere; messages, stacks and + * (nested) custom context are scrubbed for secrets and PII before send * * Payload contract: `/v1/browser` version 1 of the Autter otlp-ingester, * normally reached through the customer's same-origin relay @@ -40,6 +40,14 @@ export interface AutterBrowserOptions { captureTimings?: boolean; /** Attach the last safe click or form action to failures (default true). */ captureActions?: boolean; + /** + * Secret/PII scrubbing of messages, stacks, and context (on by default). + * Add your own patterns with `{ keys, values }` — `keys` masks whole + * context values whose key matches, `values` (use the `g` flag) masks + * matching substrings anywhere. `false` disables (not recommended; the + * relay and ingester still scrub). + */ + redact?: false | { keys?: RegExp; values?: RegExp[] }; } export type AutterSeverity = "fatal" | "error" | "warning" | "info"; @@ -117,26 +125,75 @@ function stripQuery(value: string | undefined): string | undefined { return value ? value.split("?")[0] : undefined; } -// Mini redaction — the browser twin of redactAttributes() in -// @autter/runtime-node. Custom context is free-form, so values under -// sensitive-looking keys and email-shaped strings are masked before -// anything leaves the page. Deliberately tiny: this bundle is size-capped. +// Mini redaction — the browser twin of redactText()/redactAttributes() in +// @autter/runtime-node (parity checked by test-vectors/redaction.json). +// Deliberately compact: this bundle is size-capped. const SENSITIVE_KEY_RE = - /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i; -const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; + /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|private[-_.]?key|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|(^|[^a-z])ssn($|[^a-z])|cvv|card([-_. ]?(number|num|no))?$/i; +const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; +// Vendor-prefixed keys, JWTs, Bearer/Basic credentials. Server-only shapes +// (PEM blocks, npm/GitLab/SendGrid tokens) are left to the relay/ingester. +const SECRET_RE = + /\b(sk-[\w-]{20,}|[sr]k_(live|test)_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[\w-]{30,}|(AKIA|ASIA)[0-9A-Z]{16}|eyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]{8,}|bearer\s+[\w.~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +const HEADER_RE = + /((^|[^\w-])(proxy-)?(authorization|(set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +const ASSIGN_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +const URL_CRED_RE = + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)([^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = /\b(4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(011|5\d{2}))([ -]?\d){9,15}\b/g; const MASK = "[redacted]"; +function luhn(value: string): boolean { + const digits = value.replace(/\D/g, ""); + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = +digits[digits.length - 1 - i]!; + if (i % 2) d = d * 2 > 9 ? d * 2 - 9 : d * 2; + sum += d; + } + return sum % 10 === 0; +} + +function redactOptions(): { keys?: RegExp; values?: RegExp[] } | false | undefined { + return opts ? opts.redact : undefined; +} + +/** Scrub secrets/PII embedded in one string (message, stack, URL, value). */ +export function scrubText(value: string): string { + const custom = redactOptions(); + if (custom === false) return value; + let out = String(value) + .replace(SECRET_RE, MASK) + .replace(HEADER_RE, "$1" + MASK) + .replace(ASSIGN_RE, (_m, key: string, sep: string, val: string) => + key + sep + (/^["']/.test(val) ? val[0] + MASK + val[0] : MASK)) + .replace(URL_CRED_RE, "$1" + MASK + "@") + .replace(CARD_RE, (m) => (luhn(m) ? MASK : m)) + .replace(EMAIL_RE, MASK); + for (const re of (custom && custom.values) || []) out = out.replace(re, MASK); + return out; +} + +function scrubValue(value: unknown, depth: number): unknown { + if (typeof value === "string") return scrubText(value); + if (!value || typeof value !== "object") return value; + if (depth > 5) return MASK; + if (Array.isArray(value)) return value.slice(0, 50).map((item) => scrubValue(item, depth + 1)); + return redactContext(value as Record, depth + 1); +} + export function redactContext( context: Record, + depth = 0, ): Record { + const custom = redactOptions(); + if (custom === false) return context; const out: Record = {}; for (const key in context) { - const value = context[key]; - out[key] = SENSITIVE_KEY_RE.test(key) + out[key] = SENSITIVE_KEY_RE.test(key) || (custom && custom.keys && custom.keys.test(key)) ? MASK - : typeof value === "string" - ? value.replace(EMAIL_RE, MASK) - : value; + : scrubValue(context[key], depth); } return out; } @@ -309,6 +366,9 @@ function enqueue(event: BrowserEvent, urgent?: boolean): void { } attachClient(event); // Scrub before beforeSend so the last-chance hook sees the final form. + event.message = scrubText(event.message); + if (event.stack) event.stack = scrubText(event.stack); + if (event.name) event.name = scrubText(event.name); if (event.context) event.context = redactContext(event.context); if (opts.beforeSend) { const mapped = opts.beforeSend(event); @@ -430,8 +490,8 @@ export function captureMessage( /** Report an application outcome that failed without throwing. Use a stable name. */ export function captureOutcome(name: string, message: string, context?: Record): void { - const event = baseEvent("outcome", String(message).replace(EMAIL_RE, MASK)); - event.name = String(name).replace(EMAIL_RE, MASK).slice(0, 200); + const event = baseEvent("outcome", message); + event.name = String(name).slice(0, 200); event.errorType = "OutcomeFailure"; event.severity = "error"; if (context) event.context = { ...(event.context || {}), ...context }; diff --git a/packages/runtime-browser/test/redact-e2e.test.mjs b/packages/runtime-browser/test/redact-e2e.test.mjs new file mode 100644 index 0000000..f25b916 --- /dev/null +++ b/packages/runtime-browser/test/redact-e2e.test.mjs @@ -0,0 +1,50 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { captureException, captureMessage, flush, initAutterBrowser, setContext } from "../dist/index.js"; + +test("messages, stacks and nested context are scrubbed before the beacon is sent", async () => { + const listeners = new Map(); + const sent = []; + globalThis.location = { href: "https://app.example.test/", pathname: "/", origin: "https://app.example.test" }; + globalThis.window = { fetch: async () => ({ status: 200 }), addEventListener(name, cb) { listeners.set(name, cb); } }; + globalThis.document = { addEventListener() {}, visibilityState: "visible", scripts: [] }; + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { userAgent: "", sendBeacon(_url, body) { sent.push(body); return true; } }, + }); + + initAutterBrowser({ + endpoint: "/api/autter-runtime", + service: "web", + sessionTracking: false, + captureTimings: false, + captureNetworkFailures: false, + captureActions: false, + // Customer-supplied patterns extend the built-ins. + redact: { keys: /^internal_ref$/, values: [/ORD-\d{5}/g] }, + }); + setContext({ checkout: { cardNumber: "4111111111111111", step: "pay" } }); + + const err = new Error("POST /login failed: {\"password\":\"hunter2\"} for jane@example.com"); + err.stack = `${err.name}: ${err.message}\n at submit (https://app.example.test/app.js:1:1)`; + captureException(err, { + internal_ref: "abc", + request: { headers: { authorization: "Bearer abcdefghijklmnop123" }, url: "/cb?access_token=tok123456" }, + }); + captureMessage("payment for ORD-12345 used token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM"); + listeners.get("unhandledrejection")({ reason: new Error("redis://:cachepw@cache:6379 down") }); + flush(); + + const body = await sent[0].text(); + for (const secret of ["hunter2", "jane@example.com", "4111111111111111", "abcdefghijklmnop123", + "tok123456", "ORD-12345", "eyJhbGciOiJIUzI1NiJ9", "cachepw"]) { + assert.ok(!body.includes(secret), `leaked: ${secret}`); + } + const { events } = JSON.parse(body); + assert.equal(events[0].message, 'POST /login failed: {"password":"[redacted]"} for [redacted]'); + assert.match(events[0].stack, /at submit \(https:\/\/app\.example\.test\/app\.js:1:1\)/); + assert.equal(events[0].context.internal_ref, "[redacted]"); + assert.equal(events[0].context.checkout.step, "pay"); + assert.equal(events[0].context.request.url, "/cb?access_token=[redacted]"); + assert.equal(events[2].message, "redis://[redacted]@cache:6379 down"); +}); diff --git a/packages/runtime-browser/test/redact-vectors.test.mjs b/packages/runtime-browser/test/redact-vectors.test.mjs new file mode 100644 index 0000000..41ac92f --- /dev/null +++ b/packages/runtime-browser/test/redact-vectors.test.mjs @@ -0,0 +1,38 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { redactContext, scrubText } from "../dist/index.js"; + +// Shared parity vectors — same file the Node SDK, ingester and Python +// adapter test against. `skip: ["browser"]` marks server-only shapes the +// size-capped browser bundle deliberately leaves to the relay/ingester. +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +); +const join = (value) => (Array.isArray(value) ? value.join("") : value); +const MASK = "[redacted]"; + +for (const vector of vectors.text) { + if (vector.skip?.includes("browser")) continue; + test(`scrubText vector: ${vector.name}`, () => { + assert.equal(scrubText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive context keys are masked at any depth; safe keys survive", () => { + for (const key of vectors.keys.sensitive) { + const out = redactContext({ outer: { list: [{ [key]: "raw" }] } }); + assert.equal(out.outer.list[0][key], MASK, key); + } + for (const key of vectors.keys.safe) { + assert.equal(redactContext({ outer: { [key]: "kept" } }).outer[key], "kept", key); + } +}); + +test("nested string values are scrubbed for secrets, not only emails", () => { + const out = redactContext({ + request: { url: "/api?token=abc123xyz&page=1", meta: ["Bearer abcdefghijklmnop12"] }, + }); + assert.equal(out.request.url, `/api?token=${MASK}&page=1`); + assert.deepEqual(out.request.meta, [MASK]); +}); diff --git a/packages/runtime-next/src/server.ts b/packages/runtime-next/src/server.ts index 4f8955a..4687e20 100644 --- a/packages/runtime-next/src/server.ts +++ b/packages/runtime-next/src/server.ts @@ -56,6 +56,7 @@ export { makeSafeCapture, installAutterAutoFlush, redactAttributes, + redactText, } from "@autter/runtime-node"; export type { diff --git a/packages/runtime-node/README.md b/packages/runtime-node/README.md index 1403b01..35d68a3 100644 --- a/packages/runtime-node/README.md +++ b/packages/runtime-node/README.md @@ -155,23 +155,39 @@ default 1% sampling the span contribution is negligible; if you set Tail-retained error traces don't distort this: their spans carry `autter.tail_retained` and the ingester keeps them out of span-fed rollups. -### Privacy: custom attributes are redacted before they leave the process - -The browser relay whitelist-sanitises everything a client posts — but the -server tracker accepts free-form attributes from *your* code, where a stray -`captureException(err, { "user.email": … })` would otherwise go out -verbatim. Custom attributes are therefore scrubbed at capture time by -default: - -- values that look like emails, JWTs, `sk-…`/`ghp_…`/AWS/Slack tokens, - `Bearer …` headers, or `scheme://user:pass@host` URLs are masked; +### Privacy: secrets and PII are scrubbed before they leave the process + +Error context is where secrets leak: `connect ECONNREFUSED +postgres://admin:hunter2@db…` in an exception message, a provider error +echoing an API key, a `?token=` in a request URL, a stray +`captureException(err, { "user.email": … })`. Everything the SDK exports is +therefore scrubbed by default — custom attributes, exception messages and +stack traces, span status messages, LLM call attributes and errors, +structured logs, and the browser events the relay forwards. A final pass at +export time scrubs every span regardless of who created it (HTTP +instrumentation URLs, third-party instrumentations, `recordException` in +your own code). + +- secrets inside any string are masked in place, keeping the rest of the + message/stack readable: JWTs, `Bearer`/`Basic` credentials, + `Authorization:`/`Cookie:`/`Set-Cookie:` header text, + `scheme://user:pass@host` connection strings (postgres, mysql, + mongodb+srv, redis, …), vendor keys (`sk-…`, `sk_live_…`, `ghp_…`, + `github_pat_…`, `xox?-…`, `AIza…`, `AKIA…`, …), PEM private keys, + `password=`/`?token=`/`?api_key=`-style assignments, emails, and + Luhn-valid card numbers; - attributes whose **key** looks sensitive (`password`, `token`, `secret`, - `api_key`, `authorization`, `cookie`, `ssn`, `card_number`, …) are masked - wholesale; -- non-sensitive keys and primitives pass through untouched, so grouping and - dashboards keep working. + `api_key`, `authorization`, `cookie`, `session`, `ssn`, `card_number`, …) + are masked wholesale, at any nesting depth; +- non-sensitive keys, numbers (including LLM token counts) and booleans pass + through untouched, so grouping, dashboards and cost tracking keep working. + +The same patterns run in the browser SDK, the relay, the Python adapter, +and again in the ingester before storage (shared test vectors in +`test-vectors/redaction.json`). -Disable or extend it per service: +Extend it (applies to messages, stacks and URLs too) or disable it per +service: ```ts initAutterServer({ @@ -193,7 +209,9 @@ const safe = makeSafeCapture(); safe.captureException(err, { "user.email": email }); // masked before export ``` -The raw primitive is exported too (`redactAttributes(attrs, options)`). +The raw primitives are exported too (`redactAttributes(attrs, options)`, +`redactText(text, options)`). The browser relay takes the same options as +`createBrowserRelayHandler({ ..., redact })`. This closes the server-side gap to match the browser relay's payload whitelist; it is best-effort scrubbing of obvious PII shapes, not a DLP engine — keep secrets out of attributes in the first place. diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index bc3603f..90804f8 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -31,6 +31,7 @@ export { } from "./lifecycle.js"; export { redactAttributes, + redactText, type RedactOptions, } from "./redact.js"; export { diff --git a/packages/runtime-node/src/llm-instrument.ts b/packages/runtime-node/src/llm-instrument.ts index 849be01..45bcf65 100644 --- a/packages/runtime-node/src/llm-instrument.ts +++ b/packages/runtime-node/src/llm-instrument.ts @@ -4,7 +4,7 @@ import { type Attributes, type Span, } from "@opentelemetry/api"; -import { autterLlmTracer } from "./server.js"; +import { autterLlmTracer, recordError, redactWithActive } from "./server.js"; /** * One-line LLM client auto-instrumentation: @@ -115,11 +115,9 @@ function applyUsage(span: Span, usage: UsageShape): void { } function endWithError(span: Span, err: unknown): void { - if (err instanceof Error) span.recordException(err); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + // Provider errors echo request details (API key fragments, prompt + // snippets, account emails) — recorded with message/stack scrubbed. + recordError(span, err); span.end(); } @@ -223,7 +221,7 @@ export function instrumentLlmClient( "gen_ai.system": provider, "gen_ai.request.model": request.model, ...(userId ? { "autter.user_id": userId } : {}), - ...options?.attributes, + ...redactWithActive(options?.attributes), }, }); let result: unknown; diff --git a/packages/runtime-node/src/redact-exporter.ts b/packages/runtime-node/src/redact-exporter.ts new file mode 100644 index 0000000..2f0a956 --- /dev/null +++ b/packages/runtime-node/src/redact-exporter.ts @@ -0,0 +1,81 @@ +import type { ExportResult } from "@opentelemetry/core"; +import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; +import type { Redactor } from "./redact.js"; + +/** + * Last line of defence inside the customer's process: every span is + * scrubbed on its way to the OTLP exporter, whoever produced it. Capture + * calls (captureException, withLlmCall, …) already redact what they write, + * but spans also carry data the SDK never sees at capture time: + * + * - `span.recordException(err)` from HttpInstrumentation, the customer's + * own code, or third-party instrumentations — raw message + stack; + * - `http.url` / `http.target` / `url.full` with `?token=` query strings; + * - request headers when the host enables `headersToSpanAttributes`; + * - span status messages copied from `err.message`. + * + * The span is re-wrapped, never mutated (spans are shared with other + * processors). Disabled when the host passes `redactAttributes: false`. + */ +export function redactSpan(span: ReadableSpan, redactor: Redactor): ReadableSpan { + const status = span.status.message + ? { ...span.status, message: redactor.text(span.status.message) } + : span.status; + return { + name: redactor.text(span.name), + kind: span.kind, + spanContext: () => span.spanContext(), + parentSpanId: span.parentSpanId, + startTime: span.startTime, + endTime: span.endTime, + status, + attributes: redactor.exported(span.attributes), + links: span.links, + events: span.events.map((event) => + event.attributes + ? { ...event, attributes: redactor.exported(event.attributes) } + : event, + ), + duration: span.duration, + ended: span.ended, + resource: span.resource, + instrumentationLibrary: span.instrumentationLibrary, + droppedAttributesCount: span.droppedAttributesCount, + droppedEventsCount: span.droppedEventsCount, + droppedLinksCount: span.droppedLinksCount, + }; +} + +export class RedactingSpanExporter implements SpanExporter { + constructor( + private readonly inner: SpanExporter, + private readonly redactor: () => Redactor, + ) {} + + export( + spans: ReadableSpan[], + resultCallback: (result: ExportResult) => void, + ): void { + const redactor = this.redactor(); + let scrubbed = spans; + if (redactor.enabled) { + scrubbed = []; + for (const span of spans) { + try { + scrubbed.push(redactSpan(span, redactor)); + } catch { + // Fail closed: a span we could not scrub is not exported. + } + } + } + this.inner.export(scrubbed, resultCallback); + } + + shutdown(): Promise { + return this.inner.shutdown(); + } + + forceFlush(): Promise { + return this.inner.forceFlush?.() ?? Promise.resolve(); + } +} diff --git a/packages/runtime-node/src/redact.ts b/packages/runtime-node/src/redact.ts index bd93438..e2bfec4 100644 --- a/packages/runtime-node/src/redact.ts +++ b/packages/runtime-node/src/redact.ts @@ -1,12 +1,25 @@ import type { Attributes } from "@opentelemetry/api"; /** - * Server-side attribute redaction. The browser relay whitelist-sanitises - * everything a client posts (see relay.ts) — but custom attributes handed to - * `captureException` / `captureMessage` left the process verbatim. This module - * closes that gap: emails, tokens, API keys, and credentials embedded in - * string values are masked, and attributes whose NAME looks sensitive are - * masked wholesale, before anything is exported. + * Server-side secret/PII redaction. Applied to everything the Node SDK + * exports: custom attributes, exception messages and stack traces, span + * status messages, span/event attributes set by instrumentations (HTTP URLs + * with query strings, third-party instrumentations), and structured logs. + * + * Two mechanisms: + * - KEY patterns: an attribute whose NAME looks sensitive (password, token, + * cookie, authorization, api_key, session, ssn, …) is masked wholesale, + * at any nesting depth. + * - VALUE patterns: secrets embedded inside any string (JWTs, bearer/basic + * credentials, Cookie/Authorization header text, `scheme://user:pass@` + * connection strings, vendor API keys, PEM private keys, `password=` / + * `?token=` assignments, emails, Luhn-valid card numbers) are replaced in + * place so the rest of the string — the useful part of an error message + * or stack frame — survives. + * + * The same pattern set is mirrored (and kept in parity by the shared + * test-vectors/redaction.json) in @autter/runtime-browser, the ingester, and + * adapters/python/redact.py. */ type AttrValue = @@ -24,8 +37,10 @@ export interface RedactOptions { */ additionalKeyPatterns?: (RegExp | string)[]; /** - * Extra patterns scrubbed inside string VALUES (same semantics as the - * built-in email/token patterns). Extends the built-in list. + * Extra patterns scrubbed inside string VALUES — attribute values, + * exception messages, stack traces, span status messages, URLs, log + * lines. Strings are compiled as case-insensitive regex sources. Every + * pattern is applied globally. Extends the built-in list. */ additionalValuePatterns?: (RegExp | string)[]; /** Replacement token. Default "[redacted]". */ @@ -35,50 +50,89 @@ export interface RedactOptions { * sensitive keys. Default true. */ scrubEmailValues?: boolean; + /** + * Scrub Luhn-valid payment card numbers from all string values. + * Default true. + */ + scrubCardNumbers?: boolean; } // Tested against the lower-cased KEY. Deliberately anchored where a loose // substring would over-redact ("card" must not eat "discard", -// "author" must not eat "author_id"). +// "author" must not eat "author_id", "session_id" is an attribution id). const SENSITIVE_KEY_PATTERNS: RegExp[] = [ /e-?mail/, - /pass(word|wd|phrase)|^pass$/, + /pass(word|wd|phrase)|^pass$|(^|[._-])pwd$/, /token/, /secret/, /credential/, /(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key/, - /^-?x?-?authorization$|^auth(-|_|$)|bearer/, + /authori[sz]ation|^auth(-|_|$)|bearer/, /cookie/, + /(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid/, /phone|msisdn/, - /\bssn\b|social[-_ ]?security/, + /(^|[^a-z])ssn($|[^a-z])|social[-_ ]?security/, /cvv|cvc|card([-_. ]?(number|num|no))?$/, /credit[-_.]?card/, - /connection[-_.]?string/, + /connection[-_.]?string|(^|[._-])dsn$/, /recovery[-_.]?code|\botp\b|magic[-_.]?link/, ]; -// Scrubbed INSIDE string values (matched substrings are replaced, the rest -// of the value survives — useful context like a stack frame stays readable). +// --------------------------------------------------------------------------- +// VALUE patterns — scrubbed INSIDE strings. Order matters: whole-token +// shapes first, then header/assignment context, then URLs, cards, emails. +// Every pattern is idempotent on its own mask ("[redacted]" never matches). +// --------------------------------------------------------------------------- + const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; -const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; -const OPENAI_KEY_RE = /\bsk-[A-Za-z0-9]{20,}\b/g; -const GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g; -const AWS_KEY_RE = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g; -const SLACK_TOKEN_RE = /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g; -const BEARER_RE = /\bbearer\s+[A-Za-z0-9._~+/=-]{10,}/gi; -// postgres://user:password@host — credentials gone, host kept. +/** Vendor credentials recognisable by prefix alone. */ +const PREFIXED_SECRET_RE = + /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g; +const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g; +/** `Bearer ` and `Basic ` (the latter must look encoded, so + * the phrase "basic authentication" survives). */ +const AUTH_SCHEME_RE = + /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +/** `Authorization: …`, `Cookie: …`, `Set-Cookie: …` (raw header text, JSON, + * util.inspect): the whole value goes, since cookies and auth schemes + * contain spaces and semicolons. */ +const HEADER_VALUE_RE = + /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +/** `password=…`, `"token":"…"`, `?api_key=…&` — a secret-named key followed + * by `=` or `:`. Value ends at a delimiter so the surrounding query string + * or JSON stays readable. */ +const SECRET_ASSIGNMENT_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +/** `scheme://user:pass@host` (user may be empty, pass may contain `@`) and + * long token-as-username URLs (`https://@host`). Host is kept. */ const URL_CREDENTIALS_RE = - /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi; + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = + /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g; +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; + +function luhnValid(candidate: string): boolean { + const digits = candidate.replace(/\D/g, ""); + if (digits.length < 13 || digits.length > 19) return false; + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = digits.charCodeAt(digits.length - 1 - i) - 48; + if (i % 2 === 1) { + d *= 2; + if (d > 9) d -= 9; + } + sum += d; + } + return sum % 10 === 0; +} interface CompiledRedactor { keyPatterns: RegExp[]; - valuePatterns: RegExp[]; - urlCredentials: RegExp; - privateKeyBlock: RegExp; + extraValuePatterns: RegExp[]; mask: string; scrubEmailValues: boolean; + scrubCardNumbers: boolean; } function toCaseInsensitive(pattern: RegExp | string): RegExp { @@ -87,46 +141,62 @@ function toCaseInsensitive(pattern: RegExp | string): RegExp { : new RegExp(pattern.source, pattern.flags.includes("i") ? pattern.flags : `${pattern.flags}i`); } +/** Value patterns must be global, or String#replace masks only the first hit. */ +function toGlobal(pattern: RegExp | string): RegExp { + if (typeof pattern === "string") return new RegExp(pattern, "gi"); + return pattern.global + ? pattern + : new RegExp(pattern.source, `${pattern.flags}g`); +} + function compile(options?: RedactOptions): CompiledRedactor { const keyPatterns = [...SENSITIVE_KEY_PATTERNS]; - const valuePatterns: RegExp[] = [ - JWT_RE, - OPENAI_KEY_RE, - GITHUB_TOKEN_RE, - AWS_KEY_RE, - SLACK_TOKEN_RE, - BEARER_RE, - ]; - if (options?.additionalKeyPatterns) { - for (const p of options.additionalKeyPatterns) { - keyPatterns.push(toCaseInsensitive(p)); - } + const extraValuePatterns: RegExp[] = []; + for (const p of options?.additionalKeyPatterns ?? []) { + keyPatterns.push(toCaseInsensitive(p)); } - if (options?.additionalValuePatterns) { - for (const p of options.additionalValuePatterns) { - if (!valuePatterns.some((existing) => existing.source === (typeof p === "string" ? p : p.source))) { - valuePatterns.push(typeof p === "string" ? new RegExp(p, "gi") : p); - } + for (const p of options?.additionalValuePatterns ?? []) { + const re = toGlobal(p); + if (!extraValuePatterns.some((existing) => existing.source === re.source)) { + extraValuePatterns.push(re); } } return { keyPatterns, - valuePatterns, - urlCredentials: URL_CREDENTIALS_RE, - privateKeyBlock: PRIVATE_KEY_BLOCK_RE, + extraValuePatterns, mask: options?.mask ?? "[redacted]", scrubEmailValues: options?.scrubEmailValues !== false, + scrubCardNumbers: options?.scrubCardNumbers !== false, }; } +/** Hard cap on how much of one string is scanned: stacks are capped well + * below this upstream; anything longer is truncated rather than exported + * unscanned. */ +const MAX_SCRUB_CHARS = 64 * 1024; + function redactString(value: string, r: CompiledRedactor): string { - let out = value.replace(r.privateKeyBlock, r.mask); - for (const re of r.valuePatterns) { - out = out.replace(re, r.mask); + const mask = r.mask; + let out = value.length > MAX_SCRUB_CHARS ? value.slice(0, MAX_SCRUB_CHARS) : value; + out = out + .replace(PRIVATE_KEY_BLOCK_RE, mask) + .replace(PREFIXED_SECRET_RE, mask) + .replace(JWT_RE, mask) + .replace(AUTH_SCHEME_RE, mask) + .replace(HEADER_VALUE_RE, (_m, head: string) => head + mask) + .replace(SECRET_ASSIGNMENT_RE, (_m, key: string, sep: string, val: string) => + key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + mask + val[0] : mask), + ) + .replace(URL_CREDENTIALS_RE, (_m, scheme: string) => `${scheme}${mask}@`); + if (r.scrubCardNumbers) { + out = out.replace(CARD_RE, (m) => (luhnValid(m) ? mask : m)); } - out = out.replace(r.urlCredentials, "$1" + r.mask + "@"); if (r.scrubEmailValues) { - out = out.replace(EMAIL_VALUE_RE, r.mask); + out = out.replace(EMAIL_VALUE_RE, mask); + } + for (const re of r.extraValuePatterns) { + re.lastIndex = 0; + out = out.replace(re, mask); } return out; } @@ -393,16 +463,74 @@ function redactWith( return out; } +/** + * Scrub secrets/PII embedded in one string — an error message, a stack + * trace, a URL, a log line. Same value patterns as redactAttributes(). + */ +export function redactText(text: string, options?: RedactOptions): string { + return typeof text === "string" ? redactString(text, compile(options)) : text; +} + +/** A compiled redactor: callable for attribute bags, plus string and + * export-time helpers. */ +export interface Redactor { + (attributes?: Attributes | null): Attributes; + /** Scrub one string (messages, stacks, URLs). Identity when disabled. */ + text(value: string): string; + /** + * Export-time pass over attributes already on a span (set by any + * instrumentation, not just Autter's capture calls). Strings are + * value-scrubbed, string values under sensitive KEYS are masked, and + * numbers/booleans pass untouched — usage counters such as + * `ai.usage.promptTokens` must survive for cost tracking. + */ + exported(attributes: Attributes): Attributes; + readonly enabled: boolean; +} + /** * Compile a redactor once for a hot path — initAutterServer builds one from * its options and reuses it for every capture instead of recompiling. + * `false` returns a pass-through (redaction disabled by the host). */ -export function makeRedactor( - options?: boolean | RedactOptions, -): (attributes?: Attributes | null) => Attributes { +export function makeRedactor(options?: boolean | RedactOptions): Redactor { if (options === false) { - return (attributes) => ({ ...(attributes ?? {}) }); + const passthrough = ((attributes?: Attributes | null) => ({ + ...(attributes ?? {}), + })) as Redactor; + passthrough.text = (value) => value; + passthrough.exported = (attributes) => attributes; + (passthrough as { enabled: boolean }).enabled = false; + return passthrough; } const r = compile(options === true ? undefined : options); - return (attributes) => redactWith(attributes, r); + const redactor = ((attributes?: Attributes | null) => + redactWith(attributes, r)) as Redactor; + redactor.text = (value) => + typeof value === "string" ? redactString(value, r) : value; + redactor.exported = (attributes) => { + const out: Attributes = {}; + for (const key of Object.keys(attributes)) { + const value = attributes[key]; + if (typeof value === "string") { + out[key] = r.keyPatterns.some((re) => re.test(key.toLowerCase())) + ? r.mask + : redactString(value, r); + } else if (Array.isArray(value)) { + const sensitive = r.keyPatterns.some((re) => re.test(key.toLowerCase())); + out[key] = (value as unknown[]).map((item) => + typeof item === "string" + ? sensitive + ? r.mask + : redactString(item, r) + : item, + ) as typeof value; + } else { + out[key] = value; + } + } + return out; + }; + (redactor as { enabled: boolean }).enabled = true; + return redactor; } diff --git a/packages/runtime-node/src/relay.ts b/packages/runtime-node/src/relay.ts index 3eb07d5..0482152 100644 --- a/packages/runtime-node/src/relay.ts +++ b/packages/runtime-node/src/relay.ts @@ -1,4 +1,6 @@ import type { IncomingMessage, ServerResponse } from "node:http"; +import type { Attributes } from "@opentelemetry/api"; +import { makeRedactor, type RedactOptions, type Redactor } from "./redact.js"; /** * Same-origin browser relay. The browser tracker posts to a route on the @@ -23,6 +25,15 @@ export interface RelayOptions { perIpRateLimit?: number | false; /** Called when the async forward fails (default: console.warn). */ onError?: (err: unknown) => void; + /** + * Scrub secrets/PII from browser events (message, stack, name, nested + * context) before they leave your server — the same patterns as + * `initAutterServer({ redactAttributes })`. Catches old browser SDK + * versions and anything the browser's lean redactor misses. Pass a + * RedactOptions object to add your own patterns; `false` disables. + * Default true. + */ + redact?: boolean | RedactOptions; } class IpWindow { @@ -69,7 +80,12 @@ const SEVERITIES = new Set(["fatal", "error", "warning", "info"]); // Whitelist sanitiser — anything not listed here is dropped, so a // compromised or buggy client can't smuggle cookies/DOM/bodies through the // relay. Returns null when the payload is structurally invalid. -export function sanitizeBrowserPayload(raw: unknown): object | null { +const defaultRelayRedactor = makeRedactor(true); + +export function sanitizeBrowserPayload( + raw: unknown, + redactor: Redactor = defaultRelayRedactor, +): object | null { if (typeof raw !== "object" || raw === null) return null; const p = raw as Record; if (p.version !== 1) return null; @@ -90,12 +106,16 @@ export function sanitizeBrowserPayload(raw: unknown): object | null { ? { severity: e.severity } : {}), message: - typeof e.message === "string" ? e.message.slice(0, 4000) : "", - ...(typeof e.name === "string" ? { name: e.name.slice(0, 200) } : {}), + typeof e.message === "string" + ? redactor.text(e.message.slice(0, 4000)) + : "", + ...(typeof e.name === "string" + ? { name: redactor.text(e.name.slice(0, 200)) } + : {}), ...(typeof e.durationMs === "number" && Number.isFinite(e.durationMs) ? { durationMs: Math.max(0, Math.min(120000, e.durationMs)) } : {}), ...(typeof e.stack === "string" - ? { stack: e.stack.slice(0, 32000) } + ? { stack: redactor.text(e.stack.slice(0, 32000)) } : {}), ...(typeof e.errorType === "string" ? { errorType: e.errorType.slice(0, 200) } @@ -108,8 +128,10 @@ export function sanitizeBrowserPayload(raw: unknown): object | null { ...(typeof e.route === "string" ? { route: e.route.split("?")[0]!.slice(0, 1000) } : {}), - ...(typeof e.context === "object" && e.context !== null - ? { context: e.context } + ...(typeof e.context === "object" && + e.context !== null && + !Array.isArray(e.context) + ? { context: redactor(e.context as Attributes) } : {}), }); } @@ -154,6 +176,10 @@ export function createBrowserRelayFetchHandler( opts: RelayOptions, ): (request: Request) => Promise { const maxBody = opts.maxBodyBytes ?? DEFAULT_MAX_BODY; + const redactor = + opts.redact === undefined || opts.redact === true + ? defaultRelayRedactor + : makeRedactor(opts.redact); const limiter = opts.perIpRateLimit === false ? null @@ -185,7 +211,7 @@ export function createBrowserRelayFetchHandler( status: 400, }); } - const payload = sanitizeBrowserPayload(raw); + const payload = sanitizeBrowserPayload(raw, redactor); if (!payload) { return new Response(JSON.stringify({ error: "invalid payload" }), { status: 400, @@ -211,6 +237,10 @@ export function createBrowserRelayHandler( res: ServerResponse, ) => void { const maxBody = opts.maxBodyBytes ?? DEFAULT_MAX_BODY; + const redactor = + opts.redact === undefined || opts.redact === true + ? defaultRelayRedactor + : makeRedactor(opts.redact); const limiter = opts.perIpRateLimit === false ? null @@ -227,7 +257,7 @@ export function createBrowserRelayHandler( } function handleParsed(raw: unknown, res: ServerResponse): void { - const payload = sanitizeBrowserPayload(raw); + const payload = sanitizeBrowserPayload(raw, redactor); if (!payload) { respond(res, 400, { error: "invalid payload" }); return; diff --git a/packages/runtime-node/src/server.ts b/packages/runtime-node/src/server.ts index 883b1a4..9bfc348 100644 --- a/packages/runtime-node/src/server.ts +++ b/packages/runtime-node/src/server.ts @@ -55,7 +55,9 @@ import { makeRedactor, redactAttributes, type RedactOptions, + type Redactor, } from "./redact.js"; +import { RedactingSpanExporter } from "./redact-exporter.js"; import { startMemoryMetrics } from "./memory.js"; import { configureRuntimeLogger, flushRuntimeLogs, shutdownRuntimeLogger, type RuntimeLoggingOptions } from "./logger.js"; @@ -125,12 +127,17 @@ export interface AutterServerOptions { */ autoFlush?: boolean; /** - * Mask PII/secrets in custom attributes before they leave the process: - * emails, tokens (JWT, sk-, gh_, AWS, Slack), bearer headers, - * `scheme://user:pass@` URLs, and anything whose attribute KEY looks - * sensitive (password/token/secret/cookie/…). Mirrors the browser - * relay's whitelist sanitiser. `true`/undefined = defaults; pass a - * RedactOptions object to extend the patterns; `false` disables. + * Mask secrets/PII before anything leaves the process — custom + * attributes, exception messages and stack traces, span status + * messages, URLs/query strings and attributes set by any + * instrumentation (scrubbed again at export), and structured logs: + * JWTs, bearer/basic credentials, Cookie/Authorization header text, + * `scheme://user:pass@` connection strings, vendor API keys (sk-, sk_live_, + * ghp_, xox*-, AIza, AKIA, …), PEM private keys, `password=`/`?token=` + * assignments, emails, card numbers, and anything whose attribute KEY + * looks sensitive (password/token/secret/cookie/authorization/session/…). + * `true`/undefined = defaults; pass a RedactOptions object to add your + * own key/value patterns; `false` disables (not recommended). * Default true. */ redactAttributes?: boolean | RedactOptions; @@ -585,7 +592,7 @@ async function runLlmSpan( setCost: (usd) => span.setAttribute("autter.llm.cost_usd", usd), setResponseModel, setModel: setResponseModel, - setAttributes: (attributes) => span.setAttributes(attributes), + setAttributes: (attributes) => span.setAttributes(activeRedactor(attributes)), }; try { const result = await context.with( @@ -595,14 +602,8 @@ async function runLlmSpan( span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (err) { - if (err instanceof Error) { - span.recordException(err); - span.setAttribute("error.type", err.name); - } - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + recordError(span, err); + if (err instanceof Error) span.setAttribute("error.type", err.name); throw err; } finally { span.end(); @@ -622,15 +623,10 @@ function recordLlmCall(tracer: Tracer, call: TrackedLlmCall): void { }, ); if (call.error !== undefined && call.error !== null) { + recordError(span, call.error); if (call.error instanceof Error) { - span.recordException(call.error); span.setAttribute("error.type", call.error.name); } - span.setStatus({ - code: SpanStatusCode.ERROR, - message: - call.error instanceof Error ? call.error.message : String(call.error), - }); } else { span.setStatus({ code: SpanStatusCode.OK }); } @@ -652,17 +648,42 @@ async function runWithSpan( span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (err) { - if (err instanceof Error) span.recordException(err); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + recordError(span, err); throw err; } finally { span.end(); } } +/** + * Record a thrown value on a span with its message and stack scrubbed — + * the redacting replacement for `span.recordException(err)` + an error + * status. Same wire shape as OTel's recordException (an "exception" event + * with exception.type/message/stacktrace). + * @internal + */ +export function recordError(span: Span, err: unknown): void { + const isError = err instanceof Error; + const message = activeRedactor.text(isError ? err.message : String(err)); + // Like recordException, only real Errors get an exception event; other + // thrown values keep just the (scrubbed) error status. + if (isError) { + span.addEvent("exception", { + "exception.type": err.name, + "exception.message": message, + ...(err.stack + ? { "exception.stacktrace": activeRedactor.text(err.stack) } + : {}), + }); + } + span.setStatus({ code: SpanStatusCode.ERROR, message }); +} + +/** @internal Redact attributes with the active configuration. */ +export function redactWithActive(attributes?: Attributes | null): Attributes { + return activeRedactor(attributes); +} + /** Express (4/5) assigns routing state onto the core request object; a * matched handler leaves the route template on `req.route.path` and the * mount prefix on `req.baseUrl`. */ @@ -722,7 +743,7 @@ let activeAlwaysOnProvider: BasicTracerProvider | null = null; /** Compiled attribute redactor — defaults until initAutterServer applies its * own configuration. Used by every capture path including LLM attributes. */ -let activeRedactor = makeRedactor(true); +let activeRedactor: Redactor = makeRedactor(true); export function initAutterServer(options: AutterServerOptions): AutterServer { if (active) return active; @@ -732,6 +753,16 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { "", ); const headers = { authorization: `Bearer ${options.apiKey}` }; + // Every trace exporter scrubs spans on the way out, whoever created them + // (see redact-exporter.ts). Reads the active redactor lazily so the + // configuration below applies. + const traceExporter = () => + new CountingExporter( + new RedactingSpanExporter( + new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), + () => activeRedactor, + ), + ); const environment = options.environment ?? process.env.NODE_ENV ?? "production"; // Debug mode: option OR AUTTER_DEBUG env (lifecycle seeds itself from the @@ -771,22 +802,13 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { if (options.llmTracing !== false) sampler = new LlmAwareSampler(sampler); const errorTraceBuffer = retainTraces ? new ErrorTraceRetentionProcessor( - new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - { scheduledDelayMillis: 2000 }, - ), + new BatchSpanProcessor(traceExporter(), { scheduledDelayMillis: 2000 }), retainOnError, slowThresholdMs, ) : null; - const mainSpanProcessor = new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - ); + const mainSpanProcessor = new BatchSpanProcessor(traceExporter()); const metricReader = new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter({ url: `${endpoint}/v1/metrics`, @@ -831,12 +853,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { resource, sampler: new AlwaysOnSampler(), spanProcessors: [ - new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - { scheduledDelayMillis: 2000 }, - ), + new BatchSpanProcessor(traceExporter(), { scheduledDelayMillis: 2000 }), ], }); const errorTracer = alwaysOnProvider.getTracer("autter-errors"); @@ -852,9 +869,12 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { errorTraceBuffer?.retainActiveTrace(); telemetryStats.markCaptured(); const isError = error instanceof Error; - const message = isError ? error.message : String(error); - // Redaction is applied here, not at export time: PII must not leave - // the process even if an exporter misbehaves. + // Redaction is applied here AND at export time: secrets must not + // leave the process even if an exporter misbehaves. Messages and + // stacks are scrubbed too — `connect ECONNREFUSED + // postgres://admin:hunter2@…` is the typical leak, not attributes. + const message = activeRedactor.text(isError ? error.message : String(error)); + const stackText = isError && error.stack ? activeRedactor.text(error.stack) : ""; const redacted = activeRedactor(attributes); // Prefer the live request span (the HTTP server span inside an Express @@ -874,18 +894,15 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { : errorTracer.startSpan(isError ? error.name : "Error"); span.setAttributes({ "autter.severity": "error", ...redacted }); - if (isError && error.stack) { - if (redacted["autter.handled"] === true) { - span.addEvent("exception", { - "exception.type": error.name, - "exception.message": message, - "exception.stacktrace": error.stack, - "autter.handled": true, - "autter.sampled": redacted["autter.sampled"] === true, - }); - } else { - span.recordException(error); - } + if (isError && stackText) { + span.addEvent("exception", { + "exception.type": error.name, + "exception.message": message, + "exception.stacktrace": stackText, + ...(redacted["autter.handled"] === true + ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } + : {}), + }); } else { // No usable stack — an Error thrown without one, or a non-Error // value. Synthesize the call site (minus this frame) so the @@ -898,7 +915,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { span.addEvent("exception", { "exception.type": isError ? error.name : "Error", "exception.message": message, - ...(stack ? { "exception.stacktrace": stack } : {}), + ...(stack ? { "exception.stacktrace": activeRedactor.text(stack) } : {}), ...(redacted["autter.handled"] === true ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } : {}), @@ -929,16 +946,17 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { ?.split("\n") .filter((line, i) => i === 0 || !line.includes("captureMessage")) .join("\n"); + const safeMessage = activeRedactor.text(message); const span = errorTracer.startSpan("Message", { attributes: { "autter.severity": severity, ...activeRedactor(attributes) }, }); span.addEvent("exception", { "exception.type": "Message", - "exception.message": message, - ...(stack ? { "exception.stacktrace": stack } : {}), + "exception.message": safeMessage, + ...(stack ? { "exception.stacktrace": activeRedactor.text(stack) } : {}), "autter.severity": severity, }); - span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage }); span.end(); } @@ -1046,14 +1064,14 @@ export function captureException( active.captureException(error, attributes); return; } + // Before initAutterServer the span rides the host's global provider, + // whose exporter we don't control — so scrub here, at capture time. const span = trace .getTracer("autter-errors") - .startSpan(error instanceof Error ? error.name : "Error", { attributes }); - if (error instanceof Error) span.recordException(error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: error instanceof Error ? error.message : String(error), - }); + .startSpan(error instanceof Error ? error.name : "Error", { + attributes: activeRedactor(attributes), + }); + recordError(span, error); span.end(); } @@ -1068,7 +1086,12 @@ export function withProcessSpan( attributes?: Attributes, ): Promise { if (active) return active.withProcessSpan(name, fn, attributes); - return runWithSpan(trace.getTracer("autter-processes"), name, fn, attributes); + return runWithSpan( + trace.getTracer("autter-processes"), + name, + fn, + activeRedactor(attributes), + ); } /** @@ -1103,15 +1126,16 @@ export function captureMessage( active.captureMessage(message, severity, attributes); return; } + const safeMessage = activeRedactor.text(message); const span = trace.getTracer("autter-errors").startSpan("Message", { - attributes: { "autter.severity": severity, ...attributes }, + attributes: { "autter.severity": severity, ...activeRedactor(attributes) }, }); span.addEvent("exception", { "exception.type": "Message", - "exception.message": message, + "exception.message": safeMessage, "autter.severity": severity, }); - span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage }); span.end(); } diff --git a/packages/runtime-node/test/autoflush.test.mjs b/packages/runtime-node/test/autoflush.test.mjs index edd6ed0..9596054 100644 --- a/packages/runtime-node/test/autoflush.test.mjs +++ b/packages/runtime-node/test/autoflush.test.mjs @@ -90,11 +90,34 @@ test("e2e: captured exception attributes are redacted on the wire", async () => assert.ok(!wire.includes("jane.doe@example.com"), "raw email leaked"); assert.ok(!wire.includes("supersecrettoken123456"), "raw bearer token leaked"); assert.ok(!wire.includes("eyJhbGciOiJIUzI1NiJ9"), "raw JWT leaked"); + assert.ok(!wire.includes("rawcookievalue"), "nested cookie leaked"); + + // Secrets inside exception messages, stacks, and status messages. + for (const secret of [ + "hunter2pg", + "4111 1111 1111 1111", + "4eC39HqLyjWDarjtT1zdp7dc", + "CUST-123456", // customer-supplied additionalValuePatterns + "redispass99", // thrown inside withProcessSpan + "should-not-leak", // LlmCallHandle.setAttributes + "abcdefghijklmnopqrstuvwxyz0123", // provider error echoing an sk-proj key + // Third-party span: only the export-time scrub sees these. + "foreigntoken123", + "foreignbearer12345", + "foreignpw1", + ]) { + assert.ok(!wire.includes(secret), `secret leaked on the wire: ${secret}`); + } // Mask present; non-sensitive context intact. assert.ok(wire.includes("[redacted]"), "expected mask marker"); assert.ok(wire.includes("o-1"), "non-sensitive attribute was dropped"); assert.ok(wire.includes("boom: order failed"), "exception message missing"); + assert.ok(wire.includes("connect ECONNREFUSED postgres://[redacted]@db.internal:5432/app"), + "scrubbed message should keep its non-secret context"); + assert.ok(wire.includes("page=2"), "non-secret query params should survive"); + // Usage counters on third-party LLM spans must survive for cost tracking. + assert.match(wire, /"ai\.usage\.promptTokens","value":\{"intValue":42/); // Debug mode reported exports. assert.match(err, /exported \d+ span/); diff --git a/packages/runtime-node/test/fixtures/e2e-redaction.mjs b/packages/runtime-node/test/fixtures/e2e-redaction.mjs index 8567860..e9ddf21 100644 --- a/packages/runtime-node/test/fixtures/e2e-redaction.mjs +++ b/packages/runtime-node/test/fixtures/e2e-redaction.mjs @@ -1,6 +1,8 @@ // Child fixture: FULL pipeline e2e — initAutterServer against the parent's -// local collector, capture an exception full of PII, then SIGTERM. Auto-flush -// must push the redacted span out before the process dies. +// local collector, capture exceptions full of PII/secrets (in attributes, +// messages, stacks, third-party spans), then SIGTERM. Auto-flush must push +// the redacted spans out before the process dies. +import { trace } from "@opentelemetry/api"; import { initAutterServer } from "../../dist/index.js"; const autter = initAutterServer({ @@ -9,15 +11,57 @@ const autter = initAutterServer({ service: "e2e-redaction", environment: "test", metricIntervalMs: 3_600_000, // keep metric noise out of this test + traceSampleRate: 1, + redactAttributes: { additionalValuePatterns: [/CUST-\d{6}/] }, }); -setTimeout(() => { +// Fake secrets assembled from fragments so repo secret scanners stay quiet. +const STRIPE = ["sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"].join(""); + +setTimeout(async () => { autter.captureException(new Error("boom: order failed"), { "order.id": "o-1", "user.email": "jane.doe@example.com", auth: "Bearer supersecrettoken123456", "context.jwt": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + // (OTel drops object-valued attributes on its own; redaction must + // still never pass the raw cookie through.) + request: { headers: { cookie: "sid=rawcookievalue" } }, }); + + // The classic leak: a secret inside the error MESSAGE (and so the stack). + autter.captureException( + new Error("connect ECONNREFUSED postgres://admin:hunter2pg@db.internal:5432/app"), + ); + autter.captureMessage(`charge failed for card 4111 1111 1111 1111 key ${STRIPE}`, "warning"); + autter.captureException(new Error("lookup failed for CUST-123456")); + + // A process span whose thrown error carries a secret. + await autter + .withProcessSpan("nightly-sync", () => { + throw new Error("redis://:redispass99@cache:6379 refused"); + }) + .catch(() => {}); + + // LLM provider error echoing a key. + await autter + .withLlmCall({ provider: "openai", model: "gpt-x" }, (llm) => { + llm.setAttributes({ "gen_ai.request.api_key": "should-not-leak", note: "ok" }); + throw new Error("401 Incorrect API key provided: sk-proj-abcdefghijklmnopqrstuvwxyz0123"); + }) + .catch(() => {}); + + // Spans written by someone else (an instrumentation or the app itself + // via the global tracer): only export-time scrubbing can catch these. + const foreign = trace.getTracer("third-party").startSpan("GET /callback"); + foreign.setAttributes({ + "http.url": "https://app.example.test/callback?code=abc&access_token=foreigntoken123&page=2", + "http.request.header.authorization": ["Bearer foreignbearer12345"], + "ai.usage.promptTokens": 42, + }); + foreign.recordException(new Error("upstream said password=foreignpw1 for x")); + foreign.end(); + setTimeout(() => { process.kill(process.pid, "SIGTERM"); }, 100); diff --git a/packages/runtime-node/test/redact.test.mjs b/packages/runtime-node/test/redact.test.mjs index bad04da..53f3827 100644 --- a/packages/runtime-node/test/redact.test.mjs +++ b/packages/runtime-node/test/redact.test.mjs @@ -281,3 +281,84 @@ test("handles circular references without leaking sensitive values", () => { assert.equal(out.context.nested.safe, "ok"); assert.equal(out.context.self, MASK); }); + +// --------------------------------------------------------------------------- +// Shared vectors (test-vectors/redaction.json) — the same cases run against +// the browser SDK, the ingester, and the Python adapter, so every layer +// agrees on what a secret looks like. +// --------------------------------------------------------------------------- +import { readFileSync } from "node:fs"; +import { redactText, sanitizeBrowserPayload } from "../dist/index.js"; + +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +); +const join = (value) => (Array.isArray(value) ? value.join("") : value); + +for (const vector of vectors.text) { + if (vector.skip?.includes("node")) continue; + test(`redactText vector: ${vector.name}`, () => { + assert.equal(redactText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive keys are masked case-insensitively at any depth", () => { + for (const key of vectors.keys.sensitive) { + const out = redactAttributes({ outer: { inner: [{ [key]: "raw-value" }] } }); + assert.equal(out.outer.inner[0][key], MASK, key); + } + for (const key of vectors.keys.safe) { + const out = redactAttributes({ outer: { [key]: "kept" } }); + assert.equal(out.outer[key], "kept", key); + } +}); + +test("secrets inside nested string values are scrubbed, not just keys", () => { + const out = redactAttributes({ + job: { args: ["--db", "postgres://u:pw123@db/x"], note: "Cookie: a=b" }, + }); + assert.deepEqual(out.job.args, ["--db", `postgres://${MASK}@db/x`]); + assert.equal(out.job.note, `Cookie: ${MASK}`); +}); + +test("stringified stack traces are scanned line by line", () => { + const err = new Error("auth failed: Authorization: Bearer abcdefghijklmnop123456"); + const out = redactText(err.stack); + assert.ok(!out.includes("abcdefghijklmnop123456")); + assert.match(out, /^Error: auth failed: Authorization: \[redacted\]\n\s+at /); +}); + +test("custom value patterns apply to every match even without the g flag", () => { + const out = redactText("CUST-1 and CUST-2", { + additionalValuePatterns: [/CUST-\d/], + }); + assert.equal(out, `${MASK} and ${MASK}`); +}); + +test("custom key patterns extend the built-in list", () => { + const out = redactAttributes({ tenant_ref: "t-1", other: "x" }, { + additionalKeyPatterns: [/tenant_ref/], + }); + assert.deepEqual(out, { tenant_ref: MASK, other: "x" }); +}); + +test("the browser relay scrubs message, stack, name and nested context", () => { + const payload = sanitizeBrowserPayload({ + version: 1, + service: "web", + environment: "production", + events: [{ + type: "exception", + timestamp: "2026-01-01T00:00:00.000Z", + message: "fetch failed for https://api.test/x?token=abc123secret", + stack: "Error: x\n at f (https://app.test/a.js:1:1) jane@example.com", + name: "/verify/eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + context: { form: { password: "p@ss", plan: "pro" }, note: "sk-abcdefghijklmnopqrstuvwxyz" }, + }], + }); + const [event] = payload.events; + assert.equal(event.message, `fetch failed for https://api.test/x?token=${MASK}`); + assert.ok(!event.stack.includes("jane@example.com")); + assert.equal(event.name, `/verify/${MASK}`); + assert.deepEqual(event.context, { form: { password: MASK, plan: "pro" }, note: MASK }); +}); diff --git a/test-vectors/redaction.json b/test-vectors/redaction.json new file mode 100644 index 0000000..8bdd62a --- /dev/null +++ b/test-vectors/redaction.json @@ -0,0 +1,196 @@ +{ + "$comment": "Shared secret/PII scrubbing vectors. Every redactor (runtime-node redactText, runtime-browser scrubText, the relay, the otlp-ingester scrubText, and adapters/python redact_text) must map `input` to `expect`. Strings given as arrays are joined with no separator: realistic-looking vendor tokens are split into fragments so secret scanners (GitHub push protection) don't flag the fixture. `skip` lists implementations that deliberately don't cover a case.", + "text": [ + { + "name": "JWT inside an error message", + "input": "invalid session eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c for user", + "expect": "invalid session [redacted] for user" + }, + { + "name": "Authorization: Bearer header line", + "input": "request failed: Authorization: Bearer abcdefghijklmnopqrstuvwxyz1234567890", + "expect": "request failed: Authorization: [redacted]" + }, + { + "name": "bare bearer token", + "input": "sent Bearer abcdefghijklmnopqrstuvwxyz1234567890 upstream", + "expect": "sent [redacted] upstream" + }, + { + "name": "Basic auth credentials", + "input": "auth header Basic dXNlcjpwYXNzd29yZDEyMw== rejected", + "expect": "auth header [redacted] rejected" + }, + { + "name": "'basic' as an ordinary word survives", + "input": "requires basic authentication", + "expect": "requires basic authentication" + }, + { + "name": "Cookie header keeps no cookie values", + "input": "Cookie: sid=abc123; theme=dark", + "expect": "Cookie: [redacted]" + }, + { + "name": "Set-Cookie header", + "input": "Set-Cookie: session=s3cr3tvalue; Path=/; HttpOnly", + "expect": "Set-Cookie: [redacted]" + }, + { + "name": "headers serialised as JSON", + "input": "{\"authorization\":\"Bearer abcdefghijklmnopqrstuvwxyz\",\"cookie\":\"sid=abc; x=y\",\"host\":\"api.internal\"}", + "expect": "{\"authorization\":\"[redacted]\",\"cookie\":\"[redacted]\",\"host\":\"api.internal\"}" + }, + { + "name": "postgres connection string in ECONNREFUSED message", + "input": "connect ECONNREFUSED postgres://admin:hunter2@db.internal:5432/app", + "expect": "connect ECONNREFUSED postgres://[redacted]@db.internal:5432/app" + }, + { + "name": "mongodb+srv connection string", + "input": "MongoServerError at mongodb+srv://svc:Pa55word@cluster0.abcde.mongodb.net/prod", + "expect": "MongoServerError at mongodb+srv://[redacted]@cluster0.abcde.mongodb.net/prod" + }, + { + "name": "redis URL with empty username", + "input": "redis://:hunter2@cache.internal:6379/0 unreachable", + "expect": "redis://[redacted]@cache.internal:6379/0 unreachable" + }, + { + "name": "mysql URL whose password contains @", + "input": "mysql://root:p@ss@db:3306/shop", + "expect": "mysql://[redacted]@db:3306/shop" + }, + { + "name": "AWS access key id", + "input": "using AKIAIOSFODNN7EXAMPLE for s3", + "expect": "using [redacted] for s3" + }, + { + "name": "OpenAI project key (sk-proj-)", + "input": ["key ", "sk-proj-", "abcdefghijklmnopqrstuvwxyz0123456789", " rejected"], + "expect": "key [redacted] rejected" + }, + { + "name": "Anthropic key (sk-ant-)", + "input": ["x-api-key ", "sk-ant-api03-", "abcdefghijklmnopqrstuvwxyz0123"], + "expect": "x-api-key [redacted]" + }, + { + "name": "Stripe live secret key", + "input": ["stripe ", "sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc", " failed"], + "expect": "stripe [redacted] failed" + }, + { + "name": "GitHub classic and fine-grained tokens", + "skip": ["browser"], + "input": ["ghp_", "abcdefghijklmnopqrstuvwxyz1234567890", " and ", "github_pat_", "11ABCDEFG0abcdefghijklmnopqrstuvwxyz"], + "expect": "[redacted] and [redacted]" + }, + { + "name": "Slack bot token", + "input": ["xox", "b-123456789012-abcdefghijklmnopqrstuv", " end"], + "expect": "[redacted] end" + }, + { + "name": "Google API key (AIza)", + "input": ["maps ", "AIza", "SyA1234567890abcdefghijklmnopqrstu", " quota"], + "expect": "maps [redacted] quota" + }, + { + "name": "PEM private key block", + "skip": ["browser"], + "input": "cfg -----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAabc\n-----END RSA PRIVATE KEY----- loaded", + "expect": "cfg [redacted] loaded" + }, + { + "name": "token and password in a URL query string", + "input": "GET https://api.example.test/cb?token=abc123def&page=2&password=hunter2", + "expect": "GET https://api.example.test/cb?token=[redacted]&page=2&password=[redacted]" + }, + { + "name": "api_key and access_token query params", + "input": "/v1/data?api_key=k-12345&access_token=tok_98765&x=1", + "expect": "/v1/data?api_key=[redacted]&access_token=[redacted]&x=1" + }, + { + "name": "password assignment in free text and JSON", + "input": "login failed password=hunter2 body={\"password\":\"hunter 2\",\"user\":\"bob\"}", + "expect": "login failed password=[redacted] body={\"password\":\"[redacted]\",\"user\":\"bob\"}" + }, + { + "name": "client_secret and DB_PASSWORD env style", + "input": "client_secret: abcd1234 DB_PASSWORD=s3cret!", + "expect": "client_secret: [redacted] DB_PASSWORD=[redacted]" + }, + { + "name": "JSON parse error message is left alone", + "input": "Unexpected token '<', \"