From 7d936c61b2f6f744c248421028f4b05b48db6935 Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Wed, 7 Oct 2026 23:49:31 +0530 Subject: [PATCH 01/15] Scrub secrets from error messages, stacks, and every outbound field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exception data is what reaches LLM-generated fixes, and it routinely carries secrets: `connect ECONNREFUSED postgres://admin:hunter2@…`, provider errors echoing API keys, `?token=` in request URLs, cookies and bearer tokens in serialized headers. Redaction previously covered only custom attributes (and only top-level, email-only, in the browser), so messages, stack traces, span status messages, third-party spans, relayed browser events and anything an old SDK or plain OTel sender shipped went to storage and the sink verbatim. SDKs (scrub before anything leaves the customer's process, on by default): - runtime-node: scrub exception messages/stacks/status messages in captureException/captureMessage (incl. pre-init fallbacks), process spans, withLlmCall/trackLlmCall errors, LlmCallHandle.setAttributes and instrumentLlmClient; add a RedactingSpanExporter so every exported span (HTTP URLs, headersToSpanAttributes, third-party instrumentations, app-level recordException) is scrubbed; relay scrubs browser message/stack/name/nested context (new `redact` option); new redactText - runtime-browser: scrub messages, stacks, names and nested context; new `redact: { keys, values } | false` option and scrubText export (3.43 kB -> 4.14 kB brotlied, budget 5 kB) - python adapter: new stdlib-only redact.py; caught-exception sampler scrubs tracebacks with it Patterns: add Basic auth, Authorization/Cookie/Set-Cookie header text, connection strings with empty user or `@` in the password, sk-proj/ sk-ant, Stripe, GitHub fine-grained, GitLab, npm, SendGrid, Google keys, password=/?token=/?api_key= assignments, Luhn-valid cards; keys for any *authorization header, session/sessionid/sid, pwd, dsn, bounded ssn. Custom value patterns now apply globally. Email and URL-scheme quantifiers are bounded (email was quadratic on long strings: ~1 s per 32 KB, now ~10 ms). Ingester (second line of defence for old SDKs / third-party OTLP senders, before ClickHouse and the sink that feeds LLM fixing): scrub occurrence message/stack/route/errorType, span names, deep browser context and LLM-call attributes with the shared patterns; extra patterns via AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS. Parity: test-vectors/redaction.json runs against node, browser, ingester and Python; CI now runs the Python adapter tests. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 16 ++ adapters/python/caught_exceptions.py | 13 +- adapters/python/redact.py | 198 ++++++++++++++ adapters/python/test_redact.py | 111 ++++++++ docs/ARCHITECTURE.md | 19 +- docs/CONTINUOUS-DETECTION.md | 8 + packages/otlp-ingester/README.md | 2 + packages/otlp-ingester/src/config.ts | 24 ++ packages/otlp-ingester/src/context.ts | 25 +- packages/otlp-ingester/src/llm.ts | 24 +- .../otlp-ingester/src/normalize-browser.ts | 32 +-- packages/otlp-ingester/src/normalize-otlp.ts | 5 +- packages/otlp-ingester/src/redact.test.ts | 209 +++++++++++++++ packages/otlp-ingester/src/redact.ts | 119 +++++++++ packages/otlp-ingester/src/server.ts | 10 +- packages/runtime-browser/README.md | 28 +- packages/runtime-browser/src/index.ts | 90 +++++-- .../runtime-browser/test/redact-e2e.test.mjs | 50 ++++ .../test/redact-vectors.test.mjs | 38 +++ packages/runtime-next/src/server.ts | 1 + packages/runtime-node/README.md | 50 ++-- packages/runtime-node/src/index.ts | 1 + packages/runtime-node/src/llm-instrument.ts | 12 +- packages/runtime-node/src/redact-exporter.ts | 81 ++++++ packages/runtime-node/src/redact.ts | 242 +++++++++++++----- packages/runtime-node/src/relay.ts | 46 +++- packages/runtime-node/src/server.ts | 170 ++++++------ packages/runtime-node/test/autoflush.test.mjs | 23 ++ .../test/fixtures/e2e-redaction.mjs | 50 +++- packages/runtime-node/test/redact.test.mjs | 81 ++++++ test-vectors/redaction.json | 196 ++++++++++++++ 32 files changed, 1736 insertions(+), 239 deletions(-) create mode 100644 adapters/python/redact.py create mode 100644 adapters/python/test_redact.py create mode 100644 packages/otlp-ingester/src/redact.test.ts create mode 100644 packages/otlp-ingester/src/redact.ts create mode 100644 packages/runtime-browser/test/redact-e2e.test.mjs create mode 100644 packages/runtime-browser/test/redact-vectors.test.mjs create mode 100644 packages/runtime-node/src/redact-exporter.ts create mode 100644 test-vectors/redaction.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7928ece..140c957 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,3 +19,4 @@ jobs: - run: npm run test -w @autter/runtime-browser - run: npm run test -w @autter/otlp-ingester - run: npm run size -w @autter/runtime-browser + - run: python3 -m unittest discover -s adapters/python -v diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c0cd1b..4b54180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [Unreleased] + +### Security + +- Scrub secrets and PII from exception messages, stack traces and span status messages, not only custom attributes; re-scrub every span at export so third-party instrumentations, HTTP URLs (`?token=`) and `recordException` calls are covered; scrub `LlmCallHandle.setAttributes`, `instrumentLlmClient` attributes and provider errors, and module-level captures made before `initAutterServer` (`runtime-node`). +- New value patterns everywhere: Basic auth, `Authorization:`/`Cookie:`/`Set-Cookie:` header text, connection strings with empty usernames or `@` in the password, `sk-proj-`/`sk-ant-`, Stripe, GitHub fine-grained, GitLab, npm, SendGrid and Google API keys, `password=`/`?token=`/`?api_key=` assignments, Luhn-valid card numbers. New sensitive keys: any `…authorization` header key, `session`/`sessionid`/`sid`, `pwd`, `dsn`, bounded `ssn`. Custom value patterns now apply globally (`runtime-node`, `runtime-browser`, `otlp-ingester`). +- The browser relay scrubs message, stack, name and nested context (`redact` option) (`runtime-node`). +- Browser SDK scrubs messages, stacks and nested context; new `redact: { keys, values } | false` option and `scrubText` export (`runtime-browser`). +- Ingester scrubs occurrence message/stack/route, span names, browser context (deep) and LLM-call attributes before storage and the sink webhook; extra patterns via `AUTTER_REDACT_VALUE_PATTERNS` / `AUTTER_REDACT_KEY_PATTERNS` (`otlp-ingester`). +- Python adapter: new stdlib-only `adapters/python/redact.py` (`redact_text`, `redact_attributes`, `configure`); the caught-exception sampler scrubs tracebacks with it. +- Shared parity vectors in `test-vectors/redaction.json` run against every implementation. + +### Behavior notes + +- Error messages that contained secrets now fingerprint on their scrubbed form, so such issues may regroup once (they previously split per secret value). + ## [1.3.1] - 2026-09-08 ### Fixes diff --git a/adapters/python/caught_exceptions.py b/adapters/python/caught_exceptions.py index 86de6df..1b2b481 100644 --- a/adapters/python/caught_exceptions.py +++ b/adapters/python/caught_exceptions.py @@ -4,18 +4,26 @@ ones. It has substantial overhead and must be explicitly enabled. Pass an ``emit(name, stack)`` callback that records a standard OTel exception event with ``autter.handled=true`` and ``autter.sampled=true`` on an always-on span. + +The formatted traceback includes ``str(exc)``, so it is scrubbed for secrets +and PII (connection strings, tokens, API keys, emails, …) with +:func:`redact.redact_text` before ``emit`` sees it. """ from __future__ import annotations import random -import re import sys import threading import time import traceback from collections.abc import Callable +try: # imported as part of a package + from .redact import redact_text +except ImportError: # copied next to the app as a plain module + from redact import redact_text + def install_caught_sampler( emit: Callable[[str, str], None], *, sample_rate: float = 0.01, max_per_minute: int = 10 @@ -43,8 +51,7 @@ def hook(frame, event, arg): return hook seen.add(identity) sent += 1 - stack = "".join(traceback.format_exception(exc_type, exc_value, tb, limit=8))[:8000] - stack = re.sub(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", "[redacted]", stack, flags=re.I) + stack = redact_text("".join(traceback.format_exception(exc_type, exc_value, tb, limit=8))[:8000]) try: emit(exc_type.__name__, stack) except Exception: diff --git a/adapters/python/redact.py b/adapters/python/redact.py new file mode 100644 index 0000000..2b45cb9 --- /dev/null +++ b/adapters/python/redact.py @@ -0,0 +1,198 @@ +"""Secret/PII scrubbing for Python services reporting to Autter Runtime. + +Python exceptions routinely carry secrets in their *message* (``str(exc)``) +and therefore in every formatted traceback: ``psycopg2.OperationalError: +connection to postgres://admin:hunter2@db failed``, ``401 Incorrect API key +provided: sk-...``. Scrub before anything leaves the process: + + from redact import redact_text, redact_attributes + + span.add_event("exception", { + "exception.type": type(exc).__name__, + "exception.message": redact_text(str(exc)), + "exception.stacktrace": redact_text("".join(traceback.format_exception(exc))), + }) + span.set_attributes(redact_attributes(custom_context)) + +Same patterns as ``redactText``/``redactAttributes`` in @autter/runtime-node; +parity is enforced by the shared ``test-vectors/redaction.json``. Secure by +default; extend with :func:`configure` (your own value/key patterns). The +Autter ingester scrubs again server-side as a second line of defence. + +Stdlib only. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +MASK = "[redacted]" + +_PRIVATE_KEY_BLOCK = re.compile( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)" +) +_PREFIXED_SECRET = re.compile( + r"\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}" + r"|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}" + r"|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}" + r"|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}" + r"|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})" +) +_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}") +_AUTH_SCHEME = re.compile( + r"\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}" + r"|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})", + re.I, +) +_HEADER_VALUE = re.compile( + r"((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)[\"']?\s*[:=]\s*[\"']?)[^\"'\r\n]+", + re.I | re.M, +) +_SECRET_ASSIGNMENT = re.compile( + r"(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key" + r"|credential|signature|session[_-]?id|sessionid|ssn)" + r"([\"']?\s*[:=]\s*)" + r"(\"[^\"\r\n]*\"|'[^'\r\n]*'|[^\s\"'&,;)}\]\[<>]+)", + re.I, +) +_URL_CREDENTIALS = re.compile( + r"\b([a-z][a-z0-9+.-]{0,31}://)(?:[^\s/:@\"'<>]*:[^\s/\"'<>]*|[^\s/:@\"'<>]{16,})@", re.I +) +_CARD = re.compile( + r"\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b" +) +_EMAIL = re.compile(r"[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}", re.I) + +_SENSITIVE_KEYS = [ + re.compile(p) + for p in ( + r"e-?mail", + r"pass(word|wd|phrase)|^pass$|(^|[._-])pwd$", + r"token", + r"secret", + r"credential", + r"(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key", + r"authori[sz]ation|^auth(-|_|$)|bearer", + r"cookie", + r"(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid", + r"phone|msisdn", + r"(^|[^a-z])ssn($|[^a-z])|social[-_ ]?security", + r"cvv|cvc|card([-_. ]?(number|num|no))?$", + r"credit[-_.]?card", + r"connection[-_.]?string|(^|[._-])dsn$", + r"recovery[-_.]?code|\botp\b|magic[-_.]?link", + ) +] +# Numeric GenAI usage counters keep their value even though "token" matches. +_USAGE_KEY = re.compile(r"(^|\.)(input|output|total|prompt|completion)_?tokens$|token_count$", re.I) + +_extra_values: list[re.Pattern[str]] = [] +_extra_keys: list[re.Pattern[str]] = [] +_enabled = True + +_MAX_DEPTH = 8 +_MAX_ITEMS = 1000 + + +def configure( + *, + value_patterns: Iterable[str | re.Pattern[str]] = (), + key_patterns: Iterable[str | re.Pattern[str]] = (), + enabled: bool = True, +) -> None: + """Add your own patterns on top of the built-ins (replaces earlier extras). + + ``value_patterns`` are masked wherever they occur inside strings; + ``key_patterns`` mask whole values whose key matches (case-insensitive). + ``enabled=False`` turns scrubbing off (not recommended). + """ + global _extra_values, _extra_keys, _enabled + _extra_values = [p if isinstance(p, re.Pattern) else re.compile(p, re.I) for p in value_patterns] + _extra_keys = [p if isinstance(p, re.Pattern) else re.compile(p, re.I) for p in key_patterns] + _enabled = enabled + + +def _luhn_valid(candidate: str) -> bool: + digits = [int(c) for c in candidate if c.isdigit()] + if not 13 <= len(digits) <= 19: + return False + total = 0 + for i, d in enumerate(reversed(digits)): + if i % 2 == 1: + d *= 2 + if d > 9: + d -= 9 + total += d + return total % 10 == 0 + + +def _mask_assignment(match: re.Match[str]) -> str: + key, sep, value = match.group(1), match.group(2), match.group(3) + if value[:1] in ("'", '"'): + return f"{key}{sep}{value[0]}{MASK}{value[0]}" + return f"{key}{sep}{MASK}" + + +def redact_text(value: str) -> str: + """Scrub secrets/PII embedded in one string; the rest survives.""" + if not _enabled or not isinstance(value, str) or not value: + return value + out = _PRIVATE_KEY_BLOCK.sub(MASK, value) + out = _PREFIXED_SECRET.sub(MASK, out) + out = _JWT.sub(MASK, out) + out = _AUTH_SCHEME.sub(MASK, out) + out = _HEADER_VALUE.sub(lambda m: m.group(1) + MASK, out) + out = _SECRET_ASSIGNMENT.sub(_mask_assignment, out) + out = _URL_CREDENTIALS.sub(lambda m: m.group(1) + MASK + "@", out) + out = _CARD.sub(lambda m: MASK if _luhn_valid(m.group(0)) else m.group(0), out) + out = _EMAIL.sub(MASK, out) + for pattern in _extra_values: + out = pattern.sub(MASK, out) + return out + + +def is_sensitive_key(key: str) -> bool: + lowered = key.lower() + return any(p.search(lowered) for p in _SENSITIVE_KEYS) or any(p.search(key) for p in _extra_keys) + + +def _redact_value(key: str, value: Any, depth: int) -> Any: + if isinstance(value, bool) or value is None: + return value + if isinstance(value, (int, float)): + if _USAGE_KEY.search(key) and value >= 0: + return value + return MASK if is_sensitive_key(key) else value + if is_sensitive_key(key): + return MASK + if isinstance(value, str): + return redact_text(value) + if depth >= _MAX_DEPTH: + return MASK + if isinstance(value, Mapping): + return _redact_mapping(value, depth + 1) + if isinstance(value, (list, tuple)): + return [_redact_value("", item, depth + 1) for item in list(value)[:_MAX_ITEMS]] + return redact_text(str(value)) + + +def _redact_mapping(mapping: Mapping[Any, Any], depth: int) -> dict[str, Any]: + out: dict[str, Any] = {} + for i, (key, value) in enumerate(mapping.items()): + if i >= _MAX_ITEMS: + out["__redaction_truncated__"] = MASK + break + out[str(key)] = _redact_value(str(key), value, depth) + return out + + +def redact_attributes(attributes: Mapping[str, Any] | None) -> dict[str, Any]: + """Copy of ``attributes`` with sensitive keys masked (any depth) and + secrets scrubbed inside string values. Never mutates the input.""" + if not attributes: + return {} + if not _enabled: + return dict(attributes) + return _redact_mapping(attributes, 0) diff --git a/adapters/python/test_redact.py b/adapters/python/test_redact.py new file mode 100644 index 0000000..2f42694 --- /dev/null +++ b/adapters/python/test_redact.py @@ -0,0 +1,111 @@ +"""Tests for the Python adapter's redaction (stdlib unittest, no deps). + +Run: python3 -m unittest discover -s adapters/python -v +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import redact # noqa: E402 +from caught_exceptions import install_caught_sampler # noqa: E402 +from redact import MASK, configure, redact_attributes, redact_text # noqa: E402 + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "test-vectors", "redaction.json") +with open(VECTORS_PATH, encoding="utf-8") as fh: + VECTORS = json.load(fh) + + +def _join(value): + return "".join(value) if isinstance(value, list) else value + + +class SharedVectors(unittest.TestCase): + def test_text_vectors(self): + for vector in VECTORS["text"]: + if "python" in vector.get("skip", []): + continue + with self.subTest(vector["name"]): + self.assertEqual(redact_text(_join(vector["input"])), _join(vector["expect"])) + + def test_sensitive_keys_masked_at_any_depth(self): + for key in VECTORS["keys"]["sensitive"]: + with self.subTest(key): + out = redact_attributes({"outer": {"items": [{key: "raw"}]}}) + self.assertEqual(out["outer"]["items"][0][key], MASK) + + def test_safe_keys_survive(self): + for key in VECTORS["keys"]["safe"]: + with self.subTest(key): + self.assertEqual(redact_attributes({"outer": {key: "kept"}})["outer"][key], "kept") + + +class Attributes(unittest.TestCase): + def test_nested_values_and_usage_counters(self): + out = redact_attributes( + { + "db": {"dsn": "postgres://u:p@h/db", "args": ["redis://:pw@cache:6379"]}, + "gen_ai.usage.input_tokens": 12, + "max_tokens": 100, + "note": "contact jane@example.com", + } + ) + self.assertEqual(out["db"]["dsn"], MASK) + self.assertEqual(out["db"]["args"], [f"redis://{MASK}@cache:6379"]) + self.assertEqual(out["gen_ai.usage.input_tokens"], 12) + self.assertEqual(out["max_tokens"], MASK) + self.assertEqual(out["note"], f"contact {MASK}") + + def test_does_not_mutate_input(self): + original = {"password": "x", "nested": {"token": "y"}} + redact_attributes(original) + self.assertEqual(original, {"password": "x", "nested": {"token": "y"}}) + + def test_custom_patterns(self): + configure(value_patterns=[r"CUST-\d{4}"], key_patterns=[r"^tenant_ref$"]) + try: + self.assertEqual(redact_text("CUST-1234 / CUST-9999"), f"{MASK} / {MASK}") + self.assertEqual(redact_attributes({"tenant_ref": "t"}), {"tenant_ref": MASK}) + finally: + configure() + + def test_opt_out(self): + configure(enabled=False) + try: + self.assertEqual(redact_text("password=x"), "password=x") + finally: + configure() + self.assertTrue(redact._enabled) + + +class CaughtExceptionSampler(unittest.TestCase): + def test_traceback_message_is_scrubbed_before_emit(self): + emitted: list[tuple[str, str]] = [] + + def connect(): + raise ConnectionError("could not connect to postgres://admin:hunter2@db.internal:5432/app") + + stop = install_caught_sampler(lambda name, stack: emitted.append((name, stack)), sample_rate=1.0) + try: + try: + connect() + except ConnectionError: + pass + finally: + stop() + + self.assertTrue(emitted, "sampler emitted nothing") + name, stack = emitted[0] + self.assertEqual(name, "ConnectionError") + self.assertNotIn("hunter2", stack) + self.assertIn(f"postgres://{MASK}@db.internal:5432/app", stack) + self.assertIn("in connect", stack) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c038fb7..1ad0ed1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -222,11 +222,20 @@ counters, not an analytics event store). Forbidden at the schema level (rejected/stripped): full URLs with query strings, cookies, DOM content, form values, request headers/bodies, emails. -Server-side custom attributes are guarded at the source instead: the Node -SDK masks email/token/credential-shaped values and sensitive-keyed -attributes before export (`redactAttributes`, on by default), so OTLP spans -never carry a stray `user.email` even though the OTLP schema itself accepts -free-form attributes. +Server-side telemetry is guarded at the source instead: the Node SDK masks +secret/PII-shaped values (tokens, keys, credentials in connection strings +and URLs, emails, card numbers) and sensitive-keyed attributes in +attributes, exception messages, stack traces and status messages, and +re-scrubs every span at export (`redactAttributes`, on by default). The +browser SDK, the relay and the Python adapter apply the same patterns. + +Because old SDKs and plain OTel senders scrub nothing, the ingester scrubs +again before storage and before the sink webhook (which feeds downstream +LLM-assisted fixing): occurrence message/stack/route, span names, all +attribute bags, and LLM-call attributes (`src/redact.ts`; extend with +`AUTTER_REDACT_VALUE_PATTERNS` / `AUTTER_REDACT_KEY_PATTERNS`, JSON arrays +of regex sources). Messages containing secrets therefore fingerprint on +their scrubbed form. ## Sink webhook (v1) diff --git a/docs/CONTINUOUS-DETECTION.md b/docs/CONTINUOUS-DETECTION.md index 5410629..823e5eb 100644 --- a/docs/CONTINUOUS-DETECTION.md +++ b/docs/CONTINUOUS-DETECTION.md @@ -67,6 +67,14 @@ OTel `exception` event with `autter.handled=true` and overhead, applies to the current and newly created threads, and should be enabled only for targeted diagnosis. Other languages can send the same event. +The sampler scrubs each traceback (which includes `str(exc)`) for secrets +and PII before calling `emit`, using `adapters/python/redact.py` (stdlib +only; copy it next to `caught_exceptions.py`). Use the same module for your +own `record_exception` calls — `redact_text(str(exc))`, +`redact_text(traceback_text)`, `redact_attributes(context)` — and +`configure(value_patterns=[...], key_patterns=[...])` for custom patterns. +The ingester scrubs again server-side. + ```python from opentelemetry import trace from caught_exceptions import install_caught_sampler diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 1f3a360..a993628 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -66,6 +66,8 @@ The validator webhook may return the same extra fields: | `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) | | `OCCURRENCE_TTL_DAYS` / `SPAN_TTL_DAYS` / `METRICS_TTL_DAYS` | `14` / `7` / `90` | ClickHouse TTLs (applied at table creation) | | `LLM_CALL_TTL_DAYS` | `90` | Retention for `runtime_llm_calls` rows | +| `AUTTER_REDACT_VALUE_PATTERNS` | — | JSON array of extra regex sources masked in stored/forwarded text and attribute values (built-in secret/PII patterns always apply) | +| `AUTTER_REDACT_KEY_PATTERNS` | — | JSON array of extra regex sources for attribute keys whose values are masked wholesale | ## LLM / GenAI calls diff --git a/packages/otlp-ingester/src/config.ts b/packages/otlp-ingester/src/config.ts index 45d1d5c..e64679f 100644 --- a/packages/otlp-ingester/src/config.ts +++ b/packages/otlp-ingester/src/config.ts @@ -38,6 +38,15 @@ export interface IngesterConfig { spanTtlDays: number; metricsTtlDays: number; llmCallTtlDays: number; + /** + * Extra regex sources scrubbed from stored/forwarded free text and + * attribute values, and extra regex sources for sensitive attribute + * keys — on top of the built-in secret/PII patterns (see redact.ts). + * Env: AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS (JSON + * arrays of strings). + */ + redactValuePatterns?: string[]; + redactKeyPatterns?: string[]; } function intEnv(name: string, fallback: number): number { @@ -47,6 +56,19 @@ function intEnv(name: string, fallback: number): number { return Number.isFinite(value) && value > 0 ? value : fallback; } +/** JSON array of regex sources; invalid input fails startup loudly rather + * than silently running without the operator's extra patterns. */ +function patternListEnv(name: string): string[] { + const raw = process.env[name]; + if (!raw) return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed) || !parsed.every((p) => typeof p === "string")) { + throw new Error(`${name} must be a JSON array of regex source strings`); + } + for (const source of parsed) new RegExp(source as string); + return parsed as string[]; +} + function parseIngestKeys(raw: string | undefined): StaticIngestKey[] { if (!raw) return []; try { @@ -91,6 +113,8 @@ export function loadConfig(): IngesterConfig { // LLM calls keep the metrics horizon, not the span one — cost trends // need months, and per-call volume is small next to HTTP spans. llmCallTtlDays: intEnv("LLM_CALL_TTL_DAYS", 90), + redactValuePatterns: patternListEnv("AUTTER_REDACT_VALUE_PATTERNS"), + redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"), }; if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) { throw new Error( diff --git a/packages/otlp-ingester/src/context.ts b/packages/otlp-ingester/src/context.ts index ff5e30d..df80d9e 100644 --- a/packages/otlp-ingester/src/context.ts +++ b/packages/otlp-ingester/src/context.ts @@ -1,22 +1,15 @@ +import { isSensitiveKey, scrubText } from "./redact.js"; + /** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */ export function sanitizeRuntimeContext( input: unknown, ): Record { let budget = 512; const seen = new WeakSet(); + // Shared secret/PII patterns (redact.ts), then drop URL query strings + // and fragments outright — whitelist-style privacy for free text. const scrub = (value: string) => - value - .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted]") - .replace( - /\b(?:bearer\s+[A-Za-z0-9._~+\/=~-]{10,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|(?:AKIA|ASIA)[A-Z0-9]{16})\b/gi, - "[redacted]", - ) - .replace( - /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g, - "[redacted]", - ) - .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi, "$1[redacted]@") - .replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); + scrubText(value).replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); const visit = (value: unknown, key: string, depth: number): unknown => { if (--budget < 0 || depth > 6) return "[truncated]"; if (/__proto__|constructor|prototype/i.test(key)) return undefined; @@ -25,13 +18,7 @@ export function sanitizeRuntimeContext( typeof value === "number" && Number.isFinite(value) && value >= 0; - if ( - !usage && - /password|passwd|secret|token|credential|authorization|cookie|email|phone|ssn|card[._-]?number|connection[._-]?string|api[._-]?key|private[._-]?key|request[._-]?body|response[._-]?body|headers|url[._-]?query/i.test( - key, - ) - ) - return "[redacted]"; + if (!usage && key && isSensitiveKey(key)) return "[redacted]"; if (typeof value === "string") { // Serialized custom context crosses the same privacy boundary as nested values. if (/^\s*[\[{]/.test(value)) { diff --git a/packages/otlp-ingester/src/llm.ts b/packages/otlp-ingester/src/llm.ts index a2ad074..a797c3a 100644 --- a/packages/otlp-ingester/src/llm.ts +++ b/packages/otlp-ingester/src/llm.ts @@ -1,4 +1,5 @@ import { estimateLlmCostUsd } from "./llm-pricing.js"; +import { isSensitiveKey, scrubText } from "./redact.js"; import type { RuntimeLlmCall } from "./types.js"; /** @@ -160,7 +161,7 @@ export function extractLlmCall( 200, ) : "", - userId: ( + userId: scrubText( strAttr( attrs, "autter.user_id", @@ -168,21 +169,27 @@ export function extractLlmCall( "ai.telemetry.metadata.user_id", "enduser.id", "user.id", - ) ?? "" + ) ?? "", ).slice(0, 200), - sessionId: ( + sessionId: scrubText( strAttr( attrs, "autter.session_id", "ai.telemetry.metadata.sessionId", "session.id", - ) ?? "" + ) ?? "", ).slice(0, 200), attributes: llmAttributeSubset(attrs), startedAt: facts.startedAt, }; } +/** Numeric token/usage counters are kept even though their keys contain + * "token" — cost tracking depends on them. */ +function isUsageCount(key: string, value: string): boolean { + return /tokens?$|token_?count$|usage/i.test(key) && /^\d+(\.\d+)?$/.test(value); +} + /** * Drill-down context stored as JSON alongside the typed columns: only the * LLM-relevant attribute families, size-capped so a prompt accidentally @@ -201,7 +208,14 @@ function llmAttributeSubset( ) { continue; } - const trimmed = value.length > 500 ? `${value.slice(0, 500)}…` : value; + // Prompts/completions (ai.prompt, gen_ai.prompt, …) land here: scrub + // before the size cap so a secret straddling it can't half-survive. + const safe = isUsageCount(key, value) + ? value + : isSensitiveKey(key) + ? "[redacted]" + : scrubText(value); + const trimmed = safe.length > 500 ? `${safe.slice(0, 500)}…` : safe; size += key.length + trimmed.length; if (size > 4000) break; subset[key] = trimmed; diff --git a/packages/otlp-ingester/src/normalize-browser.ts b/packages/otlp-ingester/src/normalize-browser.ts index 847d58e..a6e5d92 100644 --- a/packages/otlp-ingester/src/normalize-browser.ts +++ b/packages/otlp-ingester/src/normalize-browser.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { sanitizeRuntimeContext } from "./context.js"; import { normalizeRoute } from "./fingerprint.js"; +import { scrubText } from "./redact.js"; import { asSeverity, type RuntimeMetricPoint, @@ -65,26 +67,19 @@ const TYPE_TO_ERROR_TYPE: Record = { }; // Content-level gate for the free-form `context` bag. The schema whitelist -// above is structural; this masks obvious PII/secrets inside whatever a -// (possibly outdated) SDK still sends: values under sensitive-looking keys -// and email-shaped strings — mirroring redactAttributes() in -// @autter/runtime-node and redactContext() in @autter/runtime-browser. -const SENSITIVE_KEY_RE = - /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +// above is structural; this masks secrets/PII inside whatever a (possibly +// outdated) SDK still sends — sensitive keys and secret-shaped values at any +// depth, via the same sanitiser as OTLP attributes. Message, stack, and route +// are scrubbed centrally before fingerprinting (server.ts fingerprintAll). +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; const REDACTED = "[redacted]"; function scrubContext(context: Record): Record { - const out: Record = {}; - for (const [key, value] of Object.entries(context)) { - if (value === undefined || value === null) continue; - out[key] = SENSITIVE_KEY_RE.test(key) - ? REDACTED - : typeof value === "string" - ? value.replace(EMAIL_VALUE_RE, REDACTED) - : value; - } - return out; + return sanitizeRuntimeContext( + Object.fromEntries( + Object.entries(context).filter(([, value]) => value !== undefined && value !== null), + ), + ); } /** Default severity per event type when the SDK doesn't say. */ @@ -113,6 +108,7 @@ export function normalizeBrowserPayload( occurredAt: Date, counts: { requestCount?: number; errorCount?: number; sessionCount?: number }, ): void { + route = scrubText(route); const bucketAt = new Date( Math.floor(occurredAt.getTime() / 60_000) * 60_000, ); @@ -156,7 +152,7 @@ export function normalizeBrowserPayload( const name = event.name === "browser.longtask" ? "browser.longtask" : `browser.resource:${normalizeRoute((event.name ?? "").replace(/^browser\.resource:/, "").split("?")[0]!.replace(EMAIL_VALUE_RE, REDACTED))}`; bumpRollup(name, occurredAt, { requestCount: 1 }); - const point = rollups.get(`${name} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`); + const point = rollups.get(`${scrubText(name)} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`); if (point) point.durationSumMs += event.durationMs ?? 0; continue; } diff --git a/packages/otlp-ingester/src/normalize-otlp.ts b/packages/otlp-ingester/src/normalize-otlp.ts index e939916..09cb307 100644 --- a/packages/otlp-ingester/src/normalize-otlp.ts +++ b/packages/otlp-ingester/src/normalize-otlp.ts @@ -1,4 +1,5 @@ import { decodeOtlpAttributes, sanitizeRuntimeContext, type OtlpValue } from "./context.js"; +import { scrubText } from "./redact.js"; import { normalizeRoute } from "./fingerprint.js"; import { extractLlmCall } from "./llm.js"; import { @@ -10,7 +11,7 @@ import { type RuntimeSpanRow, } from "./types.js"; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; /** * OTLP/HTTP JSON → runtime signal. Structural types cover only the fields @@ -250,7 +251,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { traceId: span.traceId ?? "", spanId: span.spanId ?? "", parentSpanId: span.parentSpanId ?? null, - name: span.name ?? "unnamed", + name: scrubText(span.name ?? "unnamed").slice(0, 1000), kind, status: isError ? "error" : "ok", route: route ? normalizeRoute(route) : null, diff --git a/packages/otlp-ingester/src/redact.test.ts b/packages/otlp-ingester/src/redact.test.ts new file mode 100644 index 0000000..a25bffb --- /dev/null +++ b/packages/otlp-ingester/src/redact.test.ts @@ -0,0 +1,209 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { after, before, test } from "node:test"; +import type { IngesterConfig } from "./config.js"; +import { sanitizeRuntimeContext } from "./context.js"; +import { configureRedaction, scrubText } from "./redact.js"; +import { createIngesterApp } from "./server.js"; + +/** + * Server-side scrubbing is the backstop for senders that don't scrub (old + * SDKs, plain OTel SDKs in any language). Vectors are shared with the Node + * SDK, browser SDK and Python adapter (test-vectors/redaction.json). + */ + +interface Vector { + name: string; + input: string | string[]; + expect: string | string[]; + skip?: string[]; +} +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +) as { text: Vector[]; keys: { sensitive: string[]; safe: string[] } }; +const join = (value: string | string[]) => (Array.isArray(value) ? value.join("") : value); + +for (const vector of vectors.text) { + if (vector.skip?.includes("ingester")) continue; + test(`scrubText vector: ${vector.name}`, () => { + assert.equal(scrubText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive keys are masked at any depth; safe keys survive", () => { + for (const key of vectors.keys.sensitive) { + const out = sanitizeRuntimeContext({ outer: { list: [{ [key]: "raw" }] } }) as any; + assert.equal(out.outer.list[0][key], "[redacted]", key); + } + for (const key of vectors.keys.safe) { + const out = sanitizeRuntimeContext({ outer: { [key]: "kept" } }) as any; + assert.equal(out.outer[key], "kept", key); + } +}); + +test("operator-supplied patterns extend the built-ins", () => { + configureRedaction({ redactValuePatterns: ["EMP-\\d{4}"], redactKeyPatterns: ["^tenant_ref$"] }); + try { + assert.equal(scrubText("employee EMP-1234 and EMP-9999"), "employee [redacted] and [redacted]"); + assert.deepEqual(sanitizeRuntimeContext({ tenant_ref: "t1", plan: "pro" }), { + tenant_ref: "[redacted]", + plan: "pro", + }); + } finally { + configureRedaction({}); + } +}); + +// --------------------------------------------------------------------------- +// Over the wire: an unscrubbing OTLP sender → ingester → ClickHouse + sink. +// Nothing secret may reach either. +// --------------------------------------------------------------------------- + +let chStub: Server; +let sinkServer: Server; +let appServer: Server; +let ingestUrl: string; +const clickhouseBodies: string[] = []; +const sinkBodies: string[] = []; + +before(async () => { + chStub = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + clickhouseBodies.push(`${req.url}\n${body}`); + res.writeHead(200).end(); + }); + }); + await new Promise((resolve) => chStub.listen(0, resolve)); + sinkServer = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + sinkBodies.push(body); + res.writeHead(202).end(); + }); + }); + await new Promise((resolve) => sinkServer.listen(0, resolve)); + const config: IngesterConfig = { + port: 0, + clickhouseUrl: `http://127.0.0.1:${(chStub.address() as AddressInfo).port}`, + clickhouseUser: "default", + clickhousePassword: "", + clickhouseDatabase: "autter_runtime", + ingestKeys: [{ key: "test-key", orgId: "org-r", repositoryId: "repo-r" }], + keyValidatorUrl: null, + keyValidatorToken: null, + sinkUrl: `http://127.0.0.1:${(sinkServer.address() as AddressInfo).port}/sink`, + sinkToken: null, + sinkMaxAttempts: 3, + sinkMaxBufferedBatches: 10, + sinkMaxBufferedMb: 1, + maxBodyBytes: 1024 * 1024, + rateLimitPerMinute: 300, + clientRateLimitPerMinute: 120, + occurrenceTtlDays: 14, + spanTtlDays: 7, + metricsTtlDays: 90, + llmCallTtlDays: 90, + }; + const { app } = createIngesterApp(config); + appServer = app.listen(0); + await new Promise((resolve) => appServer.once("listening", resolve)); + ingestUrl = `http://127.0.0.1:${(appServer.address() as AddressInfo).port}`; +}); + +after(() => { + appServer.close(); + chStub.close(); + sinkServer.close(); +}); + +const str = (key: string, value: string) => ({ key, value: { stringValue: value } }); +const STRIPE = ["sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"].join(""); + +async function waitFor(check: () => boolean): Promise { + const deadline = Date.now() + 5000; + while (!check()) { + if (Date.now() > deadline) throw new Error("timed out"); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +test("secrets from an unscrubbing OTLP sender never reach ClickHouse or the sink", async () => { + const now = `${BigInt(Date.now()) * 1_000_000n}`; + const res = await fetch(`${ingestUrl}/v1/traces`, { + method: "POST", + headers: { authorization: "Bearer test-key", "content-type": "application/json" }, + body: JSON.stringify({ + resourceSpans: [{ + resource: { attributes: [str("service.name", "py-worker")] }, + scopeSpans: [{ + spans: [ + { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + name: "GET /reset/eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + kind: 2, + startTimeUnixNano: now, + endTimeUnixNano: now, + status: { code: 2, message: "psycopg2: postgres://admin:hunter2pg@db:5432/app" }, + attributes: [ + str("http.url", "https://api.test/cb?access_token=leakytoken99&page=2"), + str("http.request.header.cookie", "sid=rawcookie42"), + ], + events: [{ + name: "exception", + timeUnixNano: now, + attributes: [ + str("exception.type", "OperationalError"), + str("exception.message", `stripe rejected ${STRIPE} for jane@example.com`), + str("exception.stacktrace", 'Traceback (most recent call last):\n File "/app/pay.py", line 9, in charge\n stripe.api_key = "' + STRIPE + '"\nOperationalError: password=hunter3py'), + ], + }], + }, + { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b8", + name: "chat gpt-x", + kind: 3, + startTimeUnixNano: now, + endTimeUnixNano: now, + attributes: [ + str("gen_ai.system", "openai"), + str("gen_ai.request.model", "gpt-x"), + str("gen_ai.prompt", "summarise: my key is sk-proj-abcdefghijklmnopqrstuvwxyz0123"), + { key: "gen_ai.usage.input_tokens", value: { intValue: "321" } }, + str("enduser.id", "jane@example.com"), + ], + }, + ], + }], + }], + }), + }); + assert.equal(res.status, 200); + await waitFor(() => sinkBodies.length > 0); + + const everything = [...clickhouseBodies, ...sinkBodies].join("\n"); + for (const secret of [ + "hunter2pg", + "4eC39HqLyjWDarjtT1zdp7dc", + "jane@example.com", + "hunter3py", + "leakytoken99", + "rawcookie42", + "eyJhbGciOiJIUzI1NiJ9", + "abcdefghijklmnopqrstuvwxyz0123", + ]) { + assert.ok(!everything.includes(secret), `secret reached storage/sink: ${secret}`); + } + // Useful context survives: error type, scrubbed message shape, stack + // frames, and LLM token usage for cost tracking. + assert.ok(everything.includes("OperationalError")); + assert.ok(everything.includes("stripe rejected [redacted] for [redacted]")); + assert.ok(everything.includes('File \\"/app/pay.py\\", line 9, in charge')); + assert.match(everything, /321/); +}); diff --git a/packages/otlp-ingester/src/redact.ts b/packages/otlp-ingester/src/redact.ts new file mode 100644 index 0000000..c3aa926 --- /dev/null +++ b/packages/otlp-ingester/src/redact.ts @@ -0,0 +1,119 @@ +/** + * Server-side secret/PII scrubbing — the ingester's second line of defence. + * + * The SDKs scrub before anything leaves the customer's process, but the + * ingester also accepts older SDK versions and arbitrary third-party OTLP + * senders (any OTel SDK in any language) that scrub nothing. Everything this + * module touches is written to ClickHouse and forwarded to the sink webhook, + * which is where error context is later read for LLM-generated fixes — so + * scrubbing here, before storage, covers every downstream consumer. + * + * Same patterns as redactText() in @autter/runtime-node; parity is enforced + * by the shared test-vectors/redaction.json. + */ + +import type { RuntimeOccurrenceInput } from "./types.js"; + +const MASK = "[redacted]"; + +const PRIVATE_KEY_BLOCK_RE = + /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; +const PREFIXED_SECRET_RE = + /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g; +const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g; +const AUTH_SCHEME_RE = + /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +const HEADER_VALUE_RE = + /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +const SECRET_ASSIGNMENT_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +const URL_CREDENTIALS_RE = + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = + /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g; +const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; + +/** Keys whose values are masked wholesale (tested case-insensitively). */ +export const SENSITIVE_KEY_RE = + /password|passwd|passphrase|(^|[._-])pwd$|secret|token|credential|authori[sz]ation|^auth(-|_|$)|bearer|cookie|e-?mail|phone|(^|[^a-z])ssn($|[^a-z])|cvv|card[._-]?number|connection[._-]?string|(^|[._-])dsn$|(api|access|private|client|signing|encryption)[._-]?key|apikey|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid|request[._-]?body|response[._-]?body|headers|url[._-]?query/i; + +let extraValuePatterns: RegExp[] = []; +let extraKeyPatterns: RegExp[] = []; + +/** + * Deployment-specific patterns on top of the built-ins (env + * AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS, JSON arrays of + * regex sources). Server-side scrubbing has no off switch: it is the + * backstop for senders that don't scrub. + */ +export function configureRedaction(options: { + redactValuePatterns?: string[]; + redactKeyPatterns?: string[]; +}): void { + extraValuePatterns = (options.redactValuePatterns ?? []).map( + (source) => new RegExp(source, "gi"), + ); + extraKeyPatterns = (options.redactKeyPatterns ?? []).map( + (source) => new RegExp(source, "i"), + ); +} + +export function isSensitiveKey(key: string): boolean { + return SENSITIVE_KEY_RE.test(key) || extraKeyPatterns.some((re) => re.test(key)); +} + +function luhnValid(candidate: string): boolean { + const digits = candidate.replace(/\D/g, ""); + if (digits.length < 13 || digits.length > 19) return false; + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = digits.charCodeAt(digits.length - 1 - i) - 48; + if (i % 2 === 1) { + d *= 2; + if (d > 9) d -= 9; + } + sum += d; + } + return sum % 10 === 0; +} + +/** Scrub secrets/PII embedded in one string; the rest of it survives. */ +export function scrubText(value: string): string { + if (typeof value !== "string" || value === "") return value; + let out = value + .replace(PRIVATE_KEY_BLOCK_RE, MASK) + .replace(PREFIXED_SECRET_RE, MASK) + .replace(JWT_RE, MASK) + .replace(AUTH_SCHEME_RE, MASK) + .replace(HEADER_VALUE_RE, (_m, head: string) => head + MASK) + .replace(SECRET_ASSIGNMENT_RE, (_m, key: string, sep: string, val: string) => + key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + MASK + val[0] : MASK), + ) + .replace(URL_CREDENTIALS_RE, (_m, scheme: string) => `${scheme}${MASK}@`) + .replace(CARD_RE, (m) => (luhnValid(m) ? MASK : m)) + .replace(EMAIL_RE, MASK); + for (const re of extraValuePatterns) { + re.lastIndex = 0; + out = out.replace(re, MASK); + } + return out; +} + +/** Nullable convenience for optional columns. */ +export function scrubOptional(value: string | null): string | null { + return value === null ? null : scrubText(value); +} + +/** Scrub the free-text fields of an occurrence (message, stack, route, + * error type). Returns a copy. */ +export function scrubOccurrence( + occurrence: RuntimeOccurrenceInput, +): RuntimeOccurrenceInput { + return { + ...occurrence, + errorType: scrubText(occurrence.errorType), + message: scrubText(occurrence.message), + stack: scrubOptional(occurrence.stack), + route: scrubOptional(occurrence.route), + }; +} diff --git a/packages/otlp-ingester/src/server.ts b/packages/otlp-ingester/src/server.ts index 24d94c4..50e2eea 100644 --- a/packages/otlp-ingester/src/server.ts +++ b/packages/otlp-ingester/src/server.ts @@ -28,6 +28,7 @@ import { decodeProfile } from "./profiles.js"; import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js"; import { validateSourceMap } from "./source-maps.js"; import { SinkForwarder } from "./sink.js"; +import { configureRedaction, scrubOccurrence } from "./redact.js"; import type { IngestContext, RuntimeOccurrence, @@ -42,6 +43,7 @@ export interface IngesterApp { } export function createIngesterApp(config: IngesterConfig): IngesterApp { + configureRedaction(config); const store = new ClickHouseStore(config); // Fingerprinted occurrences feed the consumer's issue grouping, metric // points feed the request/error-rate rollups, LLM calls feed spend @@ -175,7 +177,13 @@ export function createIngesterApp(config: IngesterConfig): IngesterApp { ctx: IngestContext, inputs: RuntimeOccurrenceInput[], ): RuntimeOccurrence[] { - return inputs.map((input, index) => { + return inputs.map((raw, index) => { + // Second line of defence, before fingerprinting, storage, and the + // sink (which feeds LLM fix generation): scrub secrets from the + // free-text fields old SDKs and third-party OTLP senders pass + // through verbatim. Attributes were already sanitised by the + // normalisers (sanitizeRuntimeContext). + const input = scrubOccurrence(raw); const fingerprint = fingerprintOccurrence(input); return { ...input, diff --git a/packages/runtime-browser/README.md b/packages/runtime-browser/README.md index de33b5f..f00676a 100644 --- a/packages/runtime-browser/README.md +++ b/packages/runtime-browser/README.md @@ -82,7 +82,8 @@ initAutterBrowser({ | `setUser(id)` | **Opaque id only** — never an email | | `setContext(ctx)` | Attached to subsequent events | | `flush()` | Force-send the queue (also runs on page hide/unload) | -| `redactContext(ctx)` | Mask obvious PII in a context bag (applied to every event automatically) | +| `redactContext(ctx)` | Mask secrets/PII in a context bag, nested (applied to every event automatically) | +| `scrubText(text)` | Mask secrets/PII inside a string (applied to every message, stack and name automatically) | ## Batching & delivery @@ -101,9 +102,22 @@ Browser and OS are a family plus major version. Filenames and script URLs are query-stripped. For CSP blocks, the directive, blocked resource origin, script path, and a short policy hash are retained. -Custom `context` is free-form, so it is scrubbed before send: values under -sensitive-looking keys (`email`, `password`, `token`, `secret`, `auth`, -`cookie`, `api_key`, `card_number`, …) are replaced with `[redacted]`, and -email-shaped substrings are masked inside ordinary string values. This -mirrors the server SDK's `redactAttributes`; the relay and ingester apply -the same rules as defense-in-depth. +Messages, stack traces and custom `context` (at any depth) are scrubbed +before send: values under sensitive-looking keys (`email`, `password`, +`token`, `secret`, `auth`, `cookie`, `session`, `api_key`, `card_number`, …) +are replaced with `[redacted]`, and secrets inside strings — JWTs, +`Bearer`/`Basic` credentials, `Cookie:`/`Authorization:` text, connection +strings with credentials, common API keys, `?token=`/`password=` +assignments, emails, card numbers — are masked in place. This mirrors the +server SDK's `redactText`/`redactAttributes`; the relay and ingester apply +the full server-side rule set again as defense-in-depth. + +Add your own patterns (or opt out, not recommended): + +```ts +initAutterBrowser({ + ..., + redact: { keys: /^internal_ref$/, values: [/ORD-\d{5}/g] }, + // redact: false, +}); +``` diff --git a/packages/runtime-browser/src/index.ts b/packages/runtime-browser/src/index.ts index 8f84a20..2177ab9 100644 --- a/packages/runtime-browser/src/index.ts +++ b/packages/runtime-browser/src/index.ts @@ -5,8 +5,8 @@ * - zero runtime dependencies, < 5 KB gzipped (CI-enforced) * - no OTel SDK, no console patching, no DOM recording, no offline storage * - privacy by construction: pathname-only routes, no cookies / form values / - * request bodies; query strings stripped everywhere; custom context is - * scrubbed for obvious PII (emails, sensitive keys) before send + * request bodies; query strings stripped everywhere; messages, stacks and + * (nested) custom context are scrubbed for secrets and PII before send * * Payload contract: `/v1/browser` version 1 of the Autter otlp-ingester, * normally reached through the customer's same-origin relay @@ -40,6 +40,14 @@ export interface AutterBrowserOptions { captureTimings?: boolean; /** Attach the last safe click or form action to failures (default true). */ captureActions?: boolean; + /** + * Secret/PII scrubbing of messages, stacks, and context (on by default). + * Add your own patterns with `{ keys, values }` — `keys` masks whole + * context values whose key matches, `values` (use the `g` flag) masks + * matching substrings anywhere. `false` disables (not recommended; the + * relay and ingester still scrub). + */ + redact?: false | { keys?: RegExp; values?: RegExp[] }; } export type AutterSeverity = "fatal" | "error" | "warning" | "info"; @@ -117,26 +125,75 @@ function stripQuery(value: string | undefined): string | undefined { return value ? value.split("?")[0] : undefined; } -// Mini redaction — the browser twin of redactAttributes() in -// @autter/runtime-node. Custom context is free-form, so values under -// sensitive-looking keys and email-shaped strings are masked before -// anything leaves the page. Deliberately tiny: this bundle is size-capped. +// Mini redaction — the browser twin of redactText()/redactAttributes() in +// @autter/runtime-node (parity checked by test-vectors/redaction.json). +// Deliberately compact: this bundle is size-capped. const SENSITIVE_KEY_RE = - /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i; -const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; + /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|private[-_.]?key|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|(^|[^a-z])ssn($|[^a-z])|cvv|card([-_. ]?(number|num|no))?$/i; +const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; +// Vendor-prefixed keys, JWTs, Bearer/Basic credentials. Server-only shapes +// (PEM blocks, npm/GitLab/SendGrid tokens) are left to the relay/ingester. +const SECRET_RE = + /\b(sk-[\w-]{20,}|[sr]k_(live|test)_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[\w-]{30,}|(AKIA|ASIA)[0-9A-Z]{16}|eyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]{8,}|bearer\s+[\w.~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +const HEADER_RE = + /((^|[^\w-])(proxy-)?(authorization|(set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +const ASSIGN_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +const URL_CRED_RE = + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)([^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = /\b(4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(011|5\d{2}))([ -]?\d){9,15}\b/g; const MASK = "[redacted]"; +function luhn(value: string): boolean { + const digits = value.replace(/\D/g, ""); + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = +digits[digits.length - 1 - i]!; + if (i % 2) d = d * 2 > 9 ? d * 2 - 9 : d * 2; + sum += d; + } + return sum % 10 === 0; +} + +function redactOptions(): { keys?: RegExp; values?: RegExp[] } | false | undefined { + return opts ? opts.redact : undefined; +} + +/** Scrub secrets/PII embedded in one string (message, stack, URL, value). */ +export function scrubText(value: string): string { + const custom = redactOptions(); + if (custom === false) return value; + let out = String(value) + .replace(SECRET_RE, MASK) + .replace(HEADER_RE, "$1" + MASK) + .replace(ASSIGN_RE, (_m, key: string, sep: string, val: string) => + key + sep + (/^["']/.test(val) ? val[0] + MASK + val[0] : MASK)) + .replace(URL_CRED_RE, "$1" + MASK + "@") + .replace(CARD_RE, (m) => (luhn(m) ? MASK : m)) + .replace(EMAIL_RE, MASK); + for (const re of (custom && custom.values) || []) out = out.replace(re, MASK); + return out; +} + +function scrubValue(value: unknown, depth: number): unknown { + if (typeof value === "string") return scrubText(value); + if (!value || typeof value !== "object") return value; + if (depth > 5) return MASK; + if (Array.isArray(value)) return value.slice(0, 50).map((item) => scrubValue(item, depth + 1)); + return redactContext(value as Record, depth + 1); +} + export function redactContext( context: Record, + depth = 0, ): Record { + const custom = redactOptions(); + if (custom === false) return context; const out: Record = {}; for (const key in context) { - const value = context[key]; - out[key] = SENSITIVE_KEY_RE.test(key) + out[key] = SENSITIVE_KEY_RE.test(key) || (custom && custom.keys && custom.keys.test(key)) ? MASK - : typeof value === "string" - ? value.replace(EMAIL_RE, MASK) - : value; + : scrubValue(context[key], depth); } return out; } @@ -309,6 +366,9 @@ function enqueue(event: BrowserEvent, urgent?: boolean): void { } attachClient(event); // Scrub before beforeSend so the last-chance hook sees the final form. + event.message = scrubText(event.message); + if (event.stack) event.stack = scrubText(event.stack); + if (event.name) event.name = scrubText(event.name); if (event.context) event.context = redactContext(event.context); if (opts.beforeSend) { const mapped = opts.beforeSend(event); @@ -430,8 +490,8 @@ export function captureMessage( /** Report an application outcome that failed without throwing. Use a stable name. */ export function captureOutcome(name: string, message: string, context?: Record): void { - const event = baseEvent("outcome", String(message).replace(EMAIL_RE, MASK)); - event.name = String(name).replace(EMAIL_RE, MASK).slice(0, 200); + const event = baseEvent("outcome", message); + event.name = String(name).slice(0, 200); event.errorType = "OutcomeFailure"; event.severity = "error"; if (context) event.context = { ...(event.context || {}), ...context }; diff --git a/packages/runtime-browser/test/redact-e2e.test.mjs b/packages/runtime-browser/test/redact-e2e.test.mjs new file mode 100644 index 0000000..f25b916 --- /dev/null +++ b/packages/runtime-browser/test/redact-e2e.test.mjs @@ -0,0 +1,50 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { captureException, captureMessage, flush, initAutterBrowser, setContext } from "../dist/index.js"; + +test("messages, stacks and nested context are scrubbed before the beacon is sent", async () => { + const listeners = new Map(); + const sent = []; + globalThis.location = { href: "https://app.example.test/", pathname: "/", origin: "https://app.example.test" }; + globalThis.window = { fetch: async () => ({ status: 200 }), addEventListener(name, cb) { listeners.set(name, cb); } }; + globalThis.document = { addEventListener() {}, visibilityState: "visible", scripts: [] }; + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { userAgent: "", sendBeacon(_url, body) { sent.push(body); return true; } }, + }); + + initAutterBrowser({ + endpoint: "/api/autter-runtime", + service: "web", + sessionTracking: false, + captureTimings: false, + captureNetworkFailures: false, + captureActions: false, + // Customer-supplied patterns extend the built-ins. + redact: { keys: /^internal_ref$/, values: [/ORD-\d{5}/g] }, + }); + setContext({ checkout: { cardNumber: "4111111111111111", step: "pay" } }); + + const err = new Error("POST /login failed: {\"password\":\"hunter2\"} for jane@example.com"); + err.stack = `${err.name}: ${err.message}\n at submit (https://app.example.test/app.js:1:1)`; + captureException(err, { + internal_ref: "abc", + request: { headers: { authorization: "Bearer abcdefghijklmnop123" }, url: "/cb?access_token=tok123456" }, + }); + captureMessage("payment for ORD-12345 used token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM"); + listeners.get("unhandledrejection")({ reason: new Error("redis://:cachepw@cache:6379 down") }); + flush(); + + const body = await sent[0].text(); + for (const secret of ["hunter2", "jane@example.com", "4111111111111111", "abcdefghijklmnop123", + "tok123456", "ORD-12345", "eyJhbGciOiJIUzI1NiJ9", "cachepw"]) { + assert.ok(!body.includes(secret), `leaked: ${secret}`); + } + const { events } = JSON.parse(body); + assert.equal(events[0].message, 'POST /login failed: {"password":"[redacted]"} for [redacted]'); + assert.match(events[0].stack, /at submit \(https:\/\/app\.example\.test\/app\.js:1:1\)/); + assert.equal(events[0].context.internal_ref, "[redacted]"); + assert.equal(events[0].context.checkout.step, "pay"); + assert.equal(events[0].context.request.url, "/cb?access_token=[redacted]"); + assert.equal(events[2].message, "redis://[redacted]@cache:6379 down"); +}); diff --git a/packages/runtime-browser/test/redact-vectors.test.mjs b/packages/runtime-browser/test/redact-vectors.test.mjs new file mode 100644 index 0000000..41ac92f --- /dev/null +++ b/packages/runtime-browser/test/redact-vectors.test.mjs @@ -0,0 +1,38 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { redactContext, scrubText } from "../dist/index.js"; + +// Shared parity vectors — same file the Node SDK, ingester and Python +// adapter test against. `skip: ["browser"]` marks server-only shapes the +// size-capped browser bundle deliberately leaves to the relay/ingester. +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +); +const join = (value) => (Array.isArray(value) ? value.join("") : value); +const MASK = "[redacted]"; + +for (const vector of vectors.text) { + if (vector.skip?.includes("browser")) continue; + test(`scrubText vector: ${vector.name}`, () => { + assert.equal(scrubText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive context keys are masked at any depth; safe keys survive", () => { + for (const key of vectors.keys.sensitive) { + const out = redactContext({ outer: { list: [{ [key]: "raw" }] } }); + assert.equal(out.outer.list[0][key], MASK, key); + } + for (const key of vectors.keys.safe) { + assert.equal(redactContext({ outer: { [key]: "kept" } }).outer[key], "kept", key); + } +}); + +test("nested string values are scrubbed for secrets, not only emails", () => { + const out = redactContext({ + request: { url: "/api?token=abc123xyz&page=1", meta: ["Bearer abcdefghijklmnop12"] }, + }); + assert.equal(out.request.url, `/api?token=${MASK}&page=1`); + assert.deepEqual(out.request.meta, [MASK]); +}); diff --git a/packages/runtime-next/src/server.ts b/packages/runtime-next/src/server.ts index 4f8955a..4687e20 100644 --- a/packages/runtime-next/src/server.ts +++ b/packages/runtime-next/src/server.ts @@ -56,6 +56,7 @@ export { makeSafeCapture, installAutterAutoFlush, redactAttributes, + redactText, } from "@autter/runtime-node"; export type { diff --git a/packages/runtime-node/README.md b/packages/runtime-node/README.md index 1403b01..35d68a3 100644 --- a/packages/runtime-node/README.md +++ b/packages/runtime-node/README.md @@ -155,23 +155,39 @@ default 1% sampling the span contribution is negligible; if you set Tail-retained error traces don't distort this: their spans carry `autter.tail_retained` and the ingester keeps them out of span-fed rollups. -### Privacy: custom attributes are redacted before they leave the process - -The browser relay whitelist-sanitises everything a client posts — but the -server tracker accepts free-form attributes from *your* code, where a stray -`captureException(err, { "user.email": … })` would otherwise go out -verbatim. Custom attributes are therefore scrubbed at capture time by -default: - -- values that look like emails, JWTs, `sk-…`/`ghp_…`/AWS/Slack tokens, - `Bearer …` headers, or `scheme://user:pass@host` URLs are masked; +### Privacy: secrets and PII are scrubbed before they leave the process + +Error context is where secrets leak: `connect ECONNREFUSED +postgres://admin:hunter2@db…` in an exception message, a provider error +echoing an API key, a `?token=` in a request URL, a stray +`captureException(err, { "user.email": … })`. Everything the SDK exports is +therefore scrubbed by default — custom attributes, exception messages and +stack traces, span status messages, LLM call attributes and errors, +structured logs, and the browser events the relay forwards. A final pass at +export time scrubs every span regardless of who created it (HTTP +instrumentation URLs, third-party instrumentations, `recordException` in +your own code). + +- secrets inside any string are masked in place, keeping the rest of the + message/stack readable: JWTs, `Bearer`/`Basic` credentials, + `Authorization:`/`Cookie:`/`Set-Cookie:` header text, + `scheme://user:pass@host` connection strings (postgres, mysql, + mongodb+srv, redis, …), vendor keys (`sk-…`, `sk_live_…`, `ghp_…`, + `github_pat_…`, `xox?-…`, `AIza…`, `AKIA…`, …), PEM private keys, + `password=`/`?token=`/`?api_key=`-style assignments, emails, and + Luhn-valid card numbers; - attributes whose **key** looks sensitive (`password`, `token`, `secret`, - `api_key`, `authorization`, `cookie`, `ssn`, `card_number`, …) are masked - wholesale; -- non-sensitive keys and primitives pass through untouched, so grouping and - dashboards keep working. + `api_key`, `authorization`, `cookie`, `session`, `ssn`, `card_number`, …) + are masked wholesale, at any nesting depth; +- non-sensitive keys, numbers (including LLM token counts) and booleans pass + through untouched, so grouping, dashboards and cost tracking keep working. + +The same patterns run in the browser SDK, the relay, the Python adapter, +and again in the ingester before storage (shared test vectors in +`test-vectors/redaction.json`). -Disable or extend it per service: +Extend it (applies to messages, stacks and URLs too) or disable it per +service: ```ts initAutterServer({ @@ -193,7 +209,9 @@ const safe = makeSafeCapture(); safe.captureException(err, { "user.email": email }); // masked before export ``` -The raw primitive is exported too (`redactAttributes(attrs, options)`). +The raw primitives are exported too (`redactAttributes(attrs, options)`, +`redactText(text, options)`). The browser relay takes the same options as +`createBrowserRelayHandler({ ..., redact })`. This closes the server-side gap to match the browser relay's payload whitelist; it is best-effort scrubbing of obvious PII shapes, not a DLP engine — keep secrets out of attributes in the first place. diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index bc3603f..90804f8 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -31,6 +31,7 @@ export { } from "./lifecycle.js"; export { redactAttributes, + redactText, type RedactOptions, } from "./redact.js"; export { diff --git a/packages/runtime-node/src/llm-instrument.ts b/packages/runtime-node/src/llm-instrument.ts index 849be01..45bcf65 100644 --- a/packages/runtime-node/src/llm-instrument.ts +++ b/packages/runtime-node/src/llm-instrument.ts @@ -4,7 +4,7 @@ import { type Attributes, type Span, } from "@opentelemetry/api"; -import { autterLlmTracer } from "./server.js"; +import { autterLlmTracer, recordError, redactWithActive } from "./server.js"; /** * One-line LLM client auto-instrumentation: @@ -115,11 +115,9 @@ function applyUsage(span: Span, usage: UsageShape): void { } function endWithError(span: Span, err: unknown): void { - if (err instanceof Error) span.recordException(err); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + // Provider errors echo request details (API key fragments, prompt + // snippets, account emails) — recorded with message/stack scrubbed. + recordError(span, err); span.end(); } @@ -223,7 +221,7 @@ export function instrumentLlmClient( "gen_ai.system": provider, "gen_ai.request.model": request.model, ...(userId ? { "autter.user_id": userId } : {}), - ...options?.attributes, + ...redactWithActive(options?.attributes), }, }); let result: unknown; diff --git a/packages/runtime-node/src/redact-exporter.ts b/packages/runtime-node/src/redact-exporter.ts new file mode 100644 index 0000000..2f0a956 --- /dev/null +++ b/packages/runtime-node/src/redact-exporter.ts @@ -0,0 +1,81 @@ +import type { ExportResult } from "@opentelemetry/core"; +import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; +import type { Redactor } from "./redact.js"; + +/** + * Last line of defence inside the customer's process: every span is + * scrubbed on its way to the OTLP exporter, whoever produced it. Capture + * calls (captureException, withLlmCall, …) already redact what they write, + * but spans also carry data the SDK never sees at capture time: + * + * - `span.recordException(err)` from HttpInstrumentation, the customer's + * own code, or third-party instrumentations — raw message + stack; + * - `http.url` / `http.target` / `url.full` with `?token=` query strings; + * - request headers when the host enables `headersToSpanAttributes`; + * - span status messages copied from `err.message`. + * + * The span is re-wrapped, never mutated (spans are shared with other + * processors). Disabled when the host passes `redactAttributes: false`. + */ +export function redactSpan(span: ReadableSpan, redactor: Redactor): ReadableSpan { + const status = span.status.message + ? { ...span.status, message: redactor.text(span.status.message) } + : span.status; + return { + name: redactor.text(span.name), + kind: span.kind, + spanContext: () => span.spanContext(), + parentSpanId: span.parentSpanId, + startTime: span.startTime, + endTime: span.endTime, + status, + attributes: redactor.exported(span.attributes), + links: span.links, + events: span.events.map((event) => + event.attributes + ? { ...event, attributes: redactor.exported(event.attributes) } + : event, + ), + duration: span.duration, + ended: span.ended, + resource: span.resource, + instrumentationLibrary: span.instrumentationLibrary, + droppedAttributesCount: span.droppedAttributesCount, + droppedEventsCount: span.droppedEventsCount, + droppedLinksCount: span.droppedLinksCount, + }; +} + +export class RedactingSpanExporter implements SpanExporter { + constructor( + private readonly inner: SpanExporter, + private readonly redactor: () => Redactor, + ) {} + + export( + spans: ReadableSpan[], + resultCallback: (result: ExportResult) => void, + ): void { + const redactor = this.redactor(); + let scrubbed = spans; + if (redactor.enabled) { + scrubbed = []; + for (const span of spans) { + try { + scrubbed.push(redactSpan(span, redactor)); + } catch { + // Fail closed: a span we could not scrub is not exported. + } + } + } + this.inner.export(scrubbed, resultCallback); + } + + shutdown(): Promise { + return this.inner.shutdown(); + } + + forceFlush(): Promise { + return this.inner.forceFlush?.() ?? Promise.resolve(); + } +} diff --git a/packages/runtime-node/src/redact.ts b/packages/runtime-node/src/redact.ts index bd93438..e2bfec4 100644 --- a/packages/runtime-node/src/redact.ts +++ b/packages/runtime-node/src/redact.ts @@ -1,12 +1,25 @@ import type { Attributes } from "@opentelemetry/api"; /** - * Server-side attribute redaction. The browser relay whitelist-sanitises - * everything a client posts (see relay.ts) — but custom attributes handed to - * `captureException` / `captureMessage` left the process verbatim. This module - * closes that gap: emails, tokens, API keys, and credentials embedded in - * string values are masked, and attributes whose NAME looks sensitive are - * masked wholesale, before anything is exported. + * Server-side secret/PII redaction. Applied to everything the Node SDK + * exports: custom attributes, exception messages and stack traces, span + * status messages, span/event attributes set by instrumentations (HTTP URLs + * with query strings, third-party instrumentations), and structured logs. + * + * Two mechanisms: + * - KEY patterns: an attribute whose NAME looks sensitive (password, token, + * cookie, authorization, api_key, session, ssn, …) is masked wholesale, + * at any nesting depth. + * - VALUE patterns: secrets embedded inside any string (JWTs, bearer/basic + * credentials, Cookie/Authorization header text, `scheme://user:pass@` + * connection strings, vendor API keys, PEM private keys, `password=` / + * `?token=` assignments, emails, Luhn-valid card numbers) are replaced in + * place so the rest of the string — the useful part of an error message + * or stack frame — survives. + * + * The same pattern set is mirrored (and kept in parity by the shared + * test-vectors/redaction.json) in @autter/runtime-browser, the ingester, and + * adapters/python/redact.py. */ type AttrValue = @@ -24,8 +37,10 @@ export interface RedactOptions { */ additionalKeyPatterns?: (RegExp | string)[]; /** - * Extra patterns scrubbed inside string VALUES (same semantics as the - * built-in email/token patterns). Extends the built-in list. + * Extra patterns scrubbed inside string VALUES — attribute values, + * exception messages, stack traces, span status messages, URLs, log + * lines. Strings are compiled as case-insensitive regex sources. Every + * pattern is applied globally. Extends the built-in list. */ additionalValuePatterns?: (RegExp | string)[]; /** Replacement token. Default "[redacted]". */ @@ -35,50 +50,89 @@ export interface RedactOptions { * sensitive keys. Default true. */ scrubEmailValues?: boolean; + /** + * Scrub Luhn-valid payment card numbers from all string values. + * Default true. + */ + scrubCardNumbers?: boolean; } // Tested against the lower-cased KEY. Deliberately anchored where a loose // substring would over-redact ("card" must not eat "discard", -// "author" must not eat "author_id"). +// "author" must not eat "author_id", "session_id" is an attribution id). const SENSITIVE_KEY_PATTERNS: RegExp[] = [ /e-?mail/, - /pass(word|wd|phrase)|^pass$/, + /pass(word|wd|phrase)|^pass$|(^|[._-])pwd$/, /token/, /secret/, /credential/, /(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key/, - /^-?x?-?authorization$|^auth(-|_|$)|bearer/, + /authori[sz]ation|^auth(-|_|$)|bearer/, /cookie/, + /(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid/, /phone|msisdn/, - /\bssn\b|social[-_ ]?security/, + /(^|[^a-z])ssn($|[^a-z])|social[-_ ]?security/, /cvv|cvc|card([-_. ]?(number|num|no))?$/, /credit[-_.]?card/, - /connection[-_.]?string/, + /connection[-_.]?string|(^|[._-])dsn$/, /recovery[-_.]?code|\botp\b|magic[-_.]?link/, ]; -// Scrubbed INSIDE string values (matched substrings are replaced, the rest -// of the value survives — useful context like a stack frame stays readable). +// --------------------------------------------------------------------------- +// VALUE patterns — scrubbed INSIDE strings. Order matters: whole-token +// shapes first, then header/assignment context, then URLs, cards, emails. +// Every pattern is idempotent on its own mask ("[redacted]" never matches). +// --------------------------------------------------------------------------- + const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; -const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; -const OPENAI_KEY_RE = /\bsk-[A-Za-z0-9]{20,}\b/g; -const GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g; -const AWS_KEY_RE = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g; -const SLACK_TOKEN_RE = /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g; -const BEARER_RE = /\bbearer\s+[A-Za-z0-9._~+/=-]{10,}/gi; -// postgres://user:password@host — credentials gone, host kept. +/** Vendor credentials recognisable by prefix alone. */ +const PREFIXED_SECRET_RE = + /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g; +const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g; +/** `Bearer ` and `Basic ` (the latter must look encoded, so + * the phrase "basic authentication" survives). */ +const AUTH_SCHEME_RE = + /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi; +/** `Authorization: …`, `Cookie: …`, `Set-Cookie: …` (raw header text, JSON, + * util.inspect): the whole value goes, since cookies and auth schemes + * contain spaces and semicolons. */ +const HEADER_VALUE_RE = + /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim; +/** `password=…`, `"token":"…"`, `?api_key=…&` — a secret-named key followed + * by `=` or `:`. Value ends at a delimiter so the surrounding query string + * or JSON stays readable. */ +const SECRET_ASSIGNMENT_RE = + /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi; +/** `scheme://user:pass@host` (user may be empty, pass may contain `@`) and + * long token-as-username URLs (`https://@host`). Host is kept. */ const URL_CREDENTIALS_RE = - /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi; + /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi; +const CARD_RE = + /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g; +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi; + +function luhnValid(candidate: string): boolean { + const digits = candidate.replace(/\D/g, ""); + if (digits.length < 13 || digits.length > 19) return false; + let sum = 0; + for (let i = 0; i < digits.length; i++) { + let d = digits.charCodeAt(digits.length - 1 - i) - 48; + if (i % 2 === 1) { + d *= 2; + if (d > 9) d -= 9; + } + sum += d; + } + return sum % 10 === 0; +} interface CompiledRedactor { keyPatterns: RegExp[]; - valuePatterns: RegExp[]; - urlCredentials: RegExp; - privateKeyBlock: RegExp; + extraValuePatterns: RegExp[]; mask: string; scrubEmailValues: boolean; + scrubCardNumbers: boolean; } function toCaseInsensitive(pattern: RegExp | string): RegExp { @@ -87,46 +141,62 @@ function toCaseInsensitive(pattern: RegExp | string): RegExp { : new RegExp(pattern.source, pattern.flags.includes("i") ? pattern.flags : `${pattern.flags}i`); } +/** Value patterns must be global, or String#replace masks only the first hit. */ +function toGlobal(pattern: RegExp | string): RegExp { + if (typeof pattern === "string") return new RegExp(pattern, "gi"); + return pattern.global + ? pattern + : new RegExp(pattern.source, `${pattern.flags}g`); +} + function compile(options?: RedactOptions): CompiledRedactor { const keyPatterns = [...SENSITIVE_KEY_PATTERNS]; - const valuePatterns: RegExp[] = [ - JWT_RE, - OPENAI_KEY_RE, - GITHUB_TOKEN_RE, - AWS_KEY_RE, - SLACK_TOKEN_RE, - BEARER_RE, - ]; - if (options?.additionalKeyPatterns) { - for (const p of options.additionalKeyPatterns) { - keyPatterns.push(toCaseInsensitive(p)); - } + const extraValuePatterns: RegExp[] = []; + for (const p of options?.additionalKeyPatterns ?? []) { + keyPatterns.push(toCaseInsensitive(p)); } - if (options?.additionalValuePatterns) { - for (const p of options.additionalValuePatterns) { - if (!valuePatterns.some((existing) => existing.source === (typeof p === "string" ? p : p.source))) { - valuePatterns.push(typeof p === "string" ? new RegExp(p, "gi") : p); - } + for (const p of options?.additionalValuePatterns ?? []) { + const re = toGlobal(p); + if (!extraValuePatterns.some((existing) => existing.source === re.source)) { + extraValuePatterns.push(re); } } return { keyPatterns, - valuePatterns, - urlCredentials: URL_CREDENTIALS_RE, - privateKeyBlock: PRIVATE_KEY_BLOCK_RE, + extraValuePatterns, mask: options?.mask ?? "[redacted]", scrubEmailValues: options?.scrubEmailValues !== false, + scrubCardNumbers: options?.scrubCardNumbers !== false, }; } +/** Hard cap on how much of one string is scanned: stacks are capped well + * below this upstream; anything longer is truncated rather than exported + * unscanned. */ +const MAX_SCRUB_CHARS = 64 * 1024; + function redactString(value: string, r: CompiledRedactor): string { - let out = value.replace(r.privateKeyBlock, r.mask); - for (const re of r.valuePatterns) { - out = out.replace(re, r.mask); + const mask = r.mask; + let out = value.length > MAX_SCRUB_CHARS ? value.slice(0, MAX_SCRUB_CHARS) : value; + out = out + .replace(PRIVATE_KEY_BLOCK_RE, mask) + .replace(PREFIXED_SECRET_RE, mask) + .replace(JWT_RE, mask) + .replace(AUTH_SCHEME_RE, mask) + .replace(HEADER_VALUE_RE, (_m, head: string) => head + mask) + .replace(SECRET_ASSIGNMENT_RE, (_m, key: string, sep: string, val: string) => + key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + mask + val[0] : mask), + ) + .replace(URL_CREDENTIALS_RE, (_m, scheme: string) => `${scheme}${mask}@`); + if (r.scrubCardNumbers) { + out = out.replace(CARD_RE, (m) => (luhnValid(m) ? mask : m)); } - out = out.replace(r.urlCredentials, "$1" + r.mask + "@"); if (r.scrubEmailValues) { - out = out.replace(EMAIL_VALUE_RE, r.mask); + out = out.replace(EMAIL_VALUE_RE, mask); + } + for (const re of r.extraValuePatterns) { + re.lastIndex = 0; + out = out.replace(re, mask); } return out; } @@ -393,16 +463,74 @@ function redactWith( return out; } +/** + * Scrub secrets/PII embedded in one string — an error message, a stack + * trace, a URL, a log line. Same value patterns as redactAttributes(). + */ +export function redactText(text: string, options?: RedactOptions): string { + return typeof text === "string" ? redactString(text, compile(options)) : text; +} + +/** A compiled redactor: callable for attribute bags, plus string and + * export-time helpers. */ +export interface Redactor { + (attributes?: Attributes | null): Attributes; + /** Scrub one string (messages, stacks, URLs). Identity when disabled. */ + text(value: string): string; + /** + * Export-time pass over attributes already on a span (set by any + * instrumentation, not just Autter's capture calls). Strings are + * value-scrubbed, string values under sensitive KEYS are masked, and + * numbers/booleans pass untouched — usage counters such as + * `ai.usage.promptTokens` must survive for cost tracking. + */ + exported(attributes: Attributes): Attributes; + readonly enabled: boolean; +} + /** * Compile a redactor once for a hot path — initAutterServer builds one from * its options and reuses it for every capture instead of recompiling. + * `false` returns a pass-through (redaction disabled by the host). */ -export function makeRedactor( - options?: boolean | RedactOptions, -): (attributes?: Attributes | null) => Attributes { +export function makeRedactor(options?: boolean | RedactOptions): Redactor { if (options === false) { - return (attributes) => ({ ...(attributes ?? {}) }); + const passthrough = ((attributes?: Attributes | null) => ({ + ...(attributes ?? {}), + })) as Redactor; + passthrough.text = (value) => value; + passthrough.exported = (attributes) => attributes; + (passthrough as { enabled: boolean }).enabled = false; + return passthrough; } const r = compile(options === true ? undefined : options); - return (attributes) => redactWith(attributes, r); + const redactor = ((attributes?: Attributes | null) => + redactWith(attributes, r)) as Redactor; + redactor.text = (value) => + typeof value === "string" ? redactString(value, r) : value; + redactor.exported = (attributes) => { + const out: Attributes = {}; + for (const key of Object.keys(attributes)) { + const value = attributes[key]; + if (typeof value === "string") { + out[key] = r.keyPatterns.some((re) => re.test(key.toLowerCase())) + ? r.mask + : redactString(value, r); + } else if (Array.isArray(value)) { + const sensitive = r.keyPatterns.some((re) => re.test(key.toLowerCase())); + out[key] = (value as unknown[]).map((item) => + typeof item === "string" + ? sensitive + ? r.mask + : redactString(item, r) + : item, + ) as typeof value; + } else { + out[key] = value; + } + } + return out; + }; + (redactor as { enabled: boolean }).enabled = true; + return redactor; } diff --git a/packages/runtime-node/src/relay.ts b/packages/runtime-node/src/relay.ts index 3eb07d5..0482152 100644 --- a/packages/runtime-node/src/relay.ts +++ b/packages/runtime-node/src/relay.ts @@ -1,4 +1,6 @@ import type { IncomingMessage, ServerResponse } from "node:http"; +import type { Attributes } from "@opentelemetry/api"; +import { makeRedactor, type RedactOptions, type Redactor } from "./redact.js"; /** * Same-origin browser relay. The browser tracker posts to a route on the @@ -23,6 +25,15 @@ export interface RelayOptions { perIpRateLimit?: number | false; /** Called when the async forward fails (default: console.warn). */ onError?: (err: unknown) => void; + /** + * Scrub secrets/PII from browser events (message, stack, name, nested + * context) before they leave your server — the same patterns as + * `initAutterServer({ redactAttributes })`. Catches old browser SDK + * versions and anything the browser's lean redactor misses. Pass a + * RedactOptions object to add your own patterns; `false` disables. + * Default true. + */ + redact?: boolean | RedactOptions; } class IpWindow { @@ -69,7 +80,12 @@ const SEVERITIES = new Set(["fatal", "error", "warning", "info"]); // Whitelist sanitiser — anything not listed here is dropped, so a // compromised or buggy client can't smuggle cookies/DOM/bodies through the // relay. Returns null when the payload is structurally invalid. -export function sanitizeBrowserPayload(raw: unknown): object | null { +const defaultRelayRedactor = makeRedactor(true); + +export function sanitizeBrowserPayload( + raw: unknown, + redactor: Redactor = defaultRelayRedactor, +): object | null { if (typeof raw !== "object" || raw === null) return null; const p = raw as Record; if (p.version !== 1) return null; @@ -90,12 +106,16 @@ export function sanitizeBrowserPayload(raw: unknown): object | null { ? { severity: e.severity } : {}), message: - typeof e.message === "string" ? e.message.slice(0, 4000) : "", - ...(typeof e.name === "string" ? { name: e.name.slice(0, 200) } : {}), + typeof e.message === "string" + ? redactor.text(e.message.slice(0, 4000)) + : "", + ...(typeof e.name === "string" + ? { name: redactor.text(e.name.slice(0, 200)) } + : {}), ...(typeof e.durationMs === "number" && Number.isFinite(e.durationMs) ? { durationMs: Math.max(0, Math.min(120000, e.durationMs)) } : {}), ...(typeof e.stack === "string" - ? { stack: e.stack.slice(0, 32000) } + ? { stack: redactor.text(e.stack.slice(0, 32000)) } : {}), ...(typeof e.errorType === "string" ? { errorType: e.errorType.slice(0, 200) } @@ -108,8 +128,10 @@ export function sanitizeBrowserPayload(raw: unknown): object | null { ...(typeof e.route === "string" ? { route: e.route.split("?")[0]!.slice(0, 1000) } : {}), - ...(typeof e.context === "object" && e.context !== null - ? { context: e.context } + ...(typeof e.context === "object" && + e.context !== null && + !Array.isArray(e.context) + ? { context: redactor(e.context as Attributes) } : {}), }); } @@ -154,6 +176,10 @@ export function createBrowserRelayFetchHandler( opts: RelayOptions, ): (request: Request) => Promise { const maxBody = opts.maxBodyBytes ?? DEFAULT_MAX_BODY; + const redactor = + opts.redact === undefined || opts.redact === true + ? defaultRelayRedactor + : makeRedactor(opts.redact); const limiter = opts.perIpRateLimit === false ? null @@ -185,7 +211,7 @@ export function createBrowserRelayFetchHandler( status: 400, }); } - const payload = sanitizeBrowserPayload(raw); + const payload = sanitizeBrowserPayload(raw, redactor); if (!payload) { return new Response(JSON.stringify({ error: "invalid payload" }), { status: 400, @@ -211,6 +237,10 @@ export function createBrowserRelayHandler( res: ServerResponse, ) => void { const maxBody = opts.maxBodyBytes ?? DEFAULT_MAX_BODY; + const redactor = + opts.redact === undefined || opts.redact === true + ? defaultRelayRedactor + : makeRedactor(opts.redact); const limiter = opts.perIpRateLimit === false ? null @@ -227,7 +257,7 @@ export function createBrowserRelayHandler( } function handleParsed(raw: unknown, res: ServerResponse): void { - const payload = sanitizeBrowserPayload(raw); + const payload = sanitizeBrowserPayload(raw, redactor); if (!payload) { respond(res, 400, { error: "invalid payload" }); return; diff --git a/packages/runtime-node/src/server.ts b/packages/runtime-node/src/server.ts index 883b1a4..9bfc348 100644 --- a/packages/runtime-node/src/server.ts +++ b/packages/runtime-node/src/server.ts @@ -55,7 +55,9 @@ import { makeRedactor, redactAttributes, type RedactOptions, + type Redactor, } from "./redact.js"; +import { RedactingSpanExporter } from "./redact-exporter.js"; import { startMemoryMetrics } from "./memory.js"; import { configureRuntimeLogger, flushRuntimeLogs, shutdownRuntimeLogger, type RuntimeLoggingOptions } from "./logger.js"; @@ -125,12 +127,17 @@ export interface AutterServerOptions { */ autoFlush?: boolean; /** - * Mask PII/secrets in custom attributes before they leave the process: - * emails, tokens (JWT, sk-, gh_, AWS, Slack), bearer headers, - * `scheme://user:pass@` URLs, and anything whose attribute KEY looks - * sensitive (password/token/secret/cookie/…). Mirrors the browser - * relay's whitelist sanitiser. `true`/undefined = defaults; pass a - * RedactOptions object to extend the patterns; `false` disables. + * Mask secrets/PII before anything leaves the process — custom + * attributes, exception messages and stack traces, span status + * messages, URLs/query strings and attributes set by any + * instrumentation (scrubbed again at export), and structured logs: + * JWTs, bearer/basic credentials, Cookie/Authorization header text, + * `scheme://user:pass@` connection strings, vendor API keys (sk-, sk_live_, + * ghp_, xox*-, AIza, AKIA, …), PEM private keys, `password=`/`?token=` + * assignments, emails, card numbers, and anything whose attribute KEY + * looks sensitive (password/token/secret/cookie/authorization/session/…). + * `true`/undefined = defaults; pass a RedactOptions object to add your + * own key/value patterns; `false` disables (not recommended). * Default true. */ redactAttributes?: boolean | RedactOptions; @@ -585,7 +592,7 @@ async function runLlmSpan( setCost: (usd) => span.setAttribute("autter.llm.cost_usd", usd), setResponseModel, setModel: setResponseModel, - setAttributes: (attributes) => span.setAttributes(attributes), + setAttributes: (attributes) => span.setAttributes(activeRedactor(attributes)), }; try { const result = await context.with( @@ -595,14 +602,8 @@ async function runLlmSpan( span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (err) { - if (err instanceof Error) { - span.recordException(err); - span.setAttribute("error.type", err.name); - } - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + recordError(span, err); + if (err instanceof Error) span.setAttribute("error.type", err.name); throw err; } finally { span.end(); @@ -622,15 +623,10 @@ function recordLlmCall(tracer: Tracer, call: TrackedLlmCall): void { }, ); if (call.error !== undefined && call.error !== null) { + recordError(span, call.error); if (call.error instanceof Error) { - span.recordException(call.error); span.setAttribute("error.type", call.error.name); } - span.setStatus({ - code: SpanStatusCode.ERROR, - message: - call.error instanceof Error ? call.error.message : String(call.error), - }); } else { span.setStatus({ code: SpanStatusCode.OK }); } @@ -652,17 +648,42 @@ async function runWithSpan( span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (err) { - if (err instanceof Error) span.recordException(err); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); + recordError(span, err); throw err; } finally { span.end(); } } +/** + * Record a thrown value on a span with its message and stack scrubbed — + * the redacting replacement for `span.recordException(err)` + an error + * status. Same wire shape as OTel's recordException (an "exception" event + * with exception.type/message/stacktrace). + * @internal + */ +export function recordError(span: Span, err: unknown): void { + const isError = err instanceof Error; + const message = activeRedactor.text(isError ? err.message : String(err)); + // Like recordException, only real Errors get an exception event; other + // thrown values keep just the (scrubbed) error status. + if (isError) { + span.addEvent("exception", { + "exception.type": err.name, + "exception.message": message, + ...(err.stack + ? { "exception.stacktrace": activeRedactor.text(err.stack) } + : {}), + }); + } + span.setStatus({ code: SpanStatusCode.ERROR, message }); +} + +/** @internal Redact attributes with the active configuration. */ +export function redactWithActive(attributes?: Attributes | null): Attributes { + return activeRedactor(attributes); +} + /** Express (4/5) assigns routing state onto the core request object; a * matched handler leaves the route template on `req.route.path` and the * mount prefix on `req.baseUrl`. */ @@ -722,7 +743,7 @@ let activeAlwaysOnProvider: BasicTracerProvider | null = null; /** Compiled attribute redactor — defaults until initAutterServer applies its * own configuration. Used by every capture path including LLM attributes. */ -let activeRedactor = makeRedactor(true); +let activeRedactor: Redactor = makeRedactor(true); export function initAutterServer(options: AutterServerOptions): AutterServer { if (active) return active; @@ -732,6 +753,16 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { "", ); const headers = { authorization: `Bearer ${options.apiKey}` }; + // Every trace exporter scrubs spans on the way out, whoever created them + // (see redact-exporter.ts). Reads the active redactor lazily so the + // configuration below applies. + const traceExporter = () => + new CountingExporter( + new RedactingSpanExporter( + new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), + () => activeRedactor, + ), + ); const environment = options.environment ?? process.env.NODE_ENV ?? "production"; // Debug mode: option OR AUTTER_DEBUG env (lifecycle seeds itself from the @@ -771,22 +802,13 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { if (options.llmTracing !== false) sampler = new LlmAwareSampler(sampler); const errorTraceBuffer = retainTraces ? new ErrorTraceRetentionProcessor( - new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - { scheduledDelayMillis: 2000 }, - ), + new BatchSpanProcessor(traceExporter(), { scheduledDelayMillis: 2000 }), retainOnError, slowThresholdMs, ) : null; - const mainSpanProcessor = new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - ); + const mainSpanProcessor = new BatchSpanProcessor(traceExporter()); const metricReader = new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter({ url: `${endpoint}/v1/metrics`, @@ -831,12 +853,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { resource, sampler: new AlwaysOnSampler(), spanProcessors: [ - new BatchSpanProcessor( - new CountingExporter( - new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), - ), - { scheduledDelayMillis: 2000 }, - ), + new BatchSpanProcessor(traceExporter(), { scheduledDelayMillis: 2000 }), ], }); const errorTracer = alwaysOnProvider.getTracer("autter-errors"); @@ -852,9 +869,12 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { errorTraceBuffer?.retainActiveTrace(); telemetryStats.markCaptured(); const isError = error instanceof Error; - const message = isError ? error.message : String(error); - // Redaction is applied here, not at export time: PII must not leave - // the process even if an exporter misbehaves. + // Redaction is applied here AND at export time: secrets must not + // leave the process even if an exporter misbehaves. Messages and + // stacks are scrubbed too — `connect ECONNREFUSED + // postgres://admin:hunter2@…` is the typical leak, not attributes. + const message = activeRedactor.text(isError ? error.message : String(error)); + const stackText = isError && error.stack ? activeRedactor.text(error.stack) : ""; const redacted = activeRedactor(attributes); // Prefer the live request span (the HTTP server span inside an Express @@ -874,18 +894,15 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { : errorTracer.startSpan(isError ? error.name : "Error"); span.setAttributes({ "autter.severity": "error", ...redacted }); - if (isError && error.stack) { - if (redacted["autter.handled"] === true) { - span.addEvent("exception", { - "exception.type": error.name, - "exception.message": message, - "exception.stacktrace": error.stack, - "autter.handled": true, - "autter.sampled": redacted["autter.sampled"] === true, - }); - } else { - span.recordException(error); - } + if (isError && stackText) { + span.addEvent("exception", { + "exception.type": error.name, + "exception.message": message, + "exception.stacktrace": stackText, + ...(redacted["autter.handled"] === true + ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } + : {}), + }); } else { // No usable stack — an Error thrown without one, or a non-Error // value. Synthesize the call site (minus this frame) so the @@ -898,7 +915,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { span.addEvent("exception", { "exception.type": isError ? error.name : "Error", "exception.message": message, - ...(stack ? { "exception.stacktrace": stack } : {}), + ...(stack ? { "exception.stacktrace": activeRedactor.text(stack) } : {}), ...(redacted["autter.handled"] === true ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } : {}), @@ -929,16 +946,17 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { ?.split("\n") .filter((line, i) => i === 0 || !line.includes("captureMessage")) .join("\n"); + const safeMessage = activeRedactor.text(message); const span = errorTracer.startSpan("Message", { attributes: { "autter.severity": severity, ...activeRedactor(attributes) }, }); span.addEvent("exception", { "exception.type": "Message", - "exception.message": message, - ...(stack ? { "exception.stacktrace": stack } : {}), + "exception.message": safeMessage, + ...(stack ? { "exception.stacktrace": activeRedactor.text(stack) } : {}), "autter.severity": severity, }); - span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage }); span.end(); } @@ -1046,14 +1064,14 @@ export function captureException( active.captureException(error, attributes); return; } + // Before initAutterServer the span rides the host's global provider, + // whose exporter we don't control — so scrub here, at capture time. const span = trace .getTracer("autter-errors") - .startSpan(error instanceof Error ? error.name : "Error", { attributes }); - if (error instanceof Error) span.recordException(error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: error instanceof Error ? error.message : String(error), - }); + .startSpan(error instanceof Error ? error.name : "Error", { + attributes: activeRedactor(attributes), + }); + recordError(span, error); span.end(); } @@ -1068,7 +1086,12 @@ export function withProcessSpan( attributes?: Attributes, ): Promise { if (active) return active.withProcessSpan(name, fn, attributes); - return runWithSpan(trace.getTracer("autter-processes"), name, fn, attributes); + return runWithSpan( + trace.getTracer("autter-processes"), + name, + fn, + activeRedactor(attributes), + ); } /** @@ -1103,15 +1126,16 @@ export function captureMessage( active.captureMessage(message, severity, attributes); return; } + const safeMessage = activeRedactor.text(message); const span = trace.getTracer("autter-errors").startSpan("Message", { - attributes: { "autter.severity": severity, ...attributes }, + attributes: { "autter.severity": severity, ...activeRedactor(attributes) }, }); span.addEvent("exception", { "exception.type": "Message", - "exception.message": message, + "exception.message": safeMessage, "autter.severity": severity, }); - span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage }); span.end(); } diff --git a/packages/runtime-node/test/autoflush.test.mjs b/packages/runtime-node/test/autoflush.test.mjs index edd6ed0..9596054 100644 --- a/packages/runtime-node/test/autoflush.test.mjs +++ b/packages/runtime-node/test/autoflush.test.mjs @@ -90,11 +90,34 @@ test("e2e: captured exception attributes are redacted on the wire", async () => assert.ok(!wire.includes("jane.doe@example.com"), "raw email leaked"); assert.ok(!wire.includes("supersecrettoken123456"), "raw bearer token leaked"); assert.ok(!wire.includes("eyJhbGciOiJIUzI1NiJ9"), "raw JWT leaked"); + assert.ok(!wire.includes("rawcookievalue"), "nested cookie leaked"); + + // Secrets inside exception messages, stacks, and status messages. + for (const secret of [ + "hunter2pg", + "4111 1111 1111 1111", + "4eC39HqLyjWDarjtT1zdp7dc", + "CUST-123456", // customer-supplied additionalValuePatterns + "redispass99", // thrown inside withProcessSpan + "should-not-leak", // LlmCallHandle.setAttributes + "abcdefghijklmnopqrstuvwxyz0123", // provider error echoing an sk-proj key + // Third-party span: only the export-time scrub sees these. + "foreigntoken123", + "foreignbearer12345", + "foreignpw1", + ]) { + assert.ok(!wire.includes(secret), `secret leaked on the wire: ${secret}`); + } // Mask present; non-sensitive context intact. assert.ok(wire.includes("[redacted]"), "expected mask marker"); assert.ok(wire.includes("o-1"), "non-sensitive attribute was dropped"); assert.ok(wire.includes("boom: order failed"), "exception message missing"); + assert.ok(wire.includes("connect ECONNREFUSED postgres://[redacted]@db.internal:5432/app"), + "scrubbed message should keep its non-secret context"); + assert.ok(wire.includes("page=2"), "non-secret query params should survive"); + // Usage counters on third-party LLM spans must survive for cost tracking. + assert.match(wire, /"ai\.usage\.promptTokens","value":\{"intValue":42/); // Debug mode reported exports. assert.match(err, /exported \d+ span/); diff --git a/packages/runtime-node/test/fixtures/e2e-redaction.mjs b/packages/runtime-node/test/fixtures/e2e-redaction.mjs index 8567860..e9ddf21 100644 --- a/packages/runtime-node/test/fixtures/e2e-redaction.mjs +++ b/packages/runtime-node/test/fixtures/e2e-redaction.mjs @@ -1,6 +1,8 @@ // Child fixture: FULL pipeline e2e — initAutterServer against the parent's -// local collector, capture an exception full of PII, then SIGTERM. Auto-flush -// must push the redacted span out before the process dies. +// local collector, capture exceptions full of PII/secrets (in attributes, +// messages, stacks, third-party spans), then SIGTERM. Auto-flush must push +// the redacted spans out before the process dies. +import { trace } from "@opentelemetry/api"; import { initAutterServer } from "../../dist/index.js"; const autter = initAutterServer({ @@ -9,15 +11,57 @@ const autter = initAutterServer({ service: "e2e-redaction", environment: "test", metricIntervalMs: 3_600_000, // keep metric noise out of this test + traceSampleRate: 1, + redactAttributes: { additionalValuePatterns: [/CUST-\d{6}/] }, }); -setTimeout(() => { +// Fake secrets assembled from fragments so repo secret scanners stay quiet. +const STRIPE = ["sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"].join(""); + +setTimeout(async () => { autter.captureException(new Error("boom: order failed"), { "order.id": "o-1", "user.email": "jane.doe@example.com", auth: "Bearer supersecrettoken123456", "context.jwt": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + // (OTel drops object-valued attributes on its own; redaction must + // still never pass the raw cookie through.) + request: { headers: { cookie: "sid=rawcookievalue" } }, }); + + // The classic leak: a secret inside the error MESSAGE (and so the stack). + autter.captureException( + new Error("connect ECONNREFUSED postgres://admin:hunter2pg@db.internal:5432/app"), + ); + autter.captureMessage(`charge failed for card 4111 1111 1111 1111 key ${STRIPE}`, "warning"); + autter.captureException(new Error("lookup failed for CUST-123456")); + + // A process span whose thrown error carries a secret. + await autter + .withProcessSpan("nightly-sync", () => { + throw new Error("redis://:redispass99@cache:6379 refused"); + }) + .catch(() => {}); + + // LLM provider error echoing a key. + await autter + .withLlmCall({ provider: "openai", model: "gpt-x" }, (llm) => { + llm.setAttributes({ "gen_ai.request.api_key": "should-not-leak", note: "ok" }); + throw new Error("401 Incorrect API key provided: sk-proj-abcdefghijklmnopqrstuvwxyz0123"); + }) + .catch(() => {}); + + // Spans written by someone else (an instrumentation or the app itself + // via the global tracer): only export-time scrubbing can catch these. + const foreign = trace.getTracer("third-party").startSpan("GET /callback"); + foreign.setAttributes({ + "http.url": "https://app.example.test/callback?code=abc&access_token=foreigntoken123&page=2", + "http.request.header.authorization": ["Bearer foreignbearer12345"], + "ai.usage.promptTokens": 42, + }); + foreign.recordException(new Error("upstream said password=foreignpw1 for x")); + foreign.end(); + setTimeout(() => { process.kill(process.pid, "SIGTERM"); }, 100); diff --git a/packages/runtime-node/test/redact.test.mjs b/packages/runtime-node/test/redact.test.mjs index bad04da..53f3827 100644 --- a/packages/runtime-node/test/redact.test.mjs +++ b/packages/runtime-node/test/redact.test.mjs @@ -281,3 +281,84 @@ test("handles circular references without leaking sensitive values", () => { assert.equal(out.context.nested.safe, "ok"); assert.equal(out.context.self, MASK); }); + +// --------------------------------------------------------------------------- +// Shared vectors (test-vectors/redaction.json) — the same cases run against +// the browser SDK, the ingester, and the Python adapter, so every layer +// agrees on what a secret looks like. +// --------------------------------------------------------------------------- +import { readFileSync } from "node:fs"; +import { redactText, sanitizeBrowserPayload } from "../dist/index.js"; + +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +); +const join = (value) => (Array.isArray(value) ? value.join("") : value); + +for (const vector of vectors.text) { + if (vector.skip?.includes("node")) continue; + test(`redactText vector: ${vector.name}`, () => { + assert.equal(redactText(join(vector.input)), join(vector.expect)); + }); +} + +test("sensitive keys are masked case-insensitively at any depth", () => { + for (const key of vectors.keys.sensitive) { + const out = redactAttributes({ outer: { inner: [{ [key]: "raw-value" }] } }); + assert.equal(out.outer.inner[0][key], MASK, key); + } + for (const key of vectors.keys.safe) { + const out = redactAttributes({ outer: { [key]: "kept" } }); + assert.equal(out.outer[key], "kept", key); + } +}); + +test("secrets inside nested string values are scrubbed, not just keys", () => { + const out = redactAttributes({ + job: { args: ["--db", "postgres://u:pw123@db/x"], note: "Cookie: a=b" }, + }); + assert.deepEqual(out.job.args, ["--db", `postgres://${MASK}@db/x`]); + assert.equal(out.job.note, `Cookie: ${MASK}`); +}); + +test("stringified stack traces are scanned line by line", () => { + const err = new Error("auth failed: Authorization: Bearer abcdefghijklmnop123456"); + const out = redactText(err.stack); + assert.ok(!out.includes("abcdefghijklmnop123456")); + assert.match(out, /^Error: auth failed: Authorization: \[redacted\]\n\s+at /); +}); + +test("custom value patterns apply to every match even without the g flag", () => { + const out = redactText("CUST-1 and CUST-2", { + additionalValuePatterns: [/CUST-\d/], + }); + assert.equal(out, `${MASK} and ${MASK}`); +}); + +test("custom key patterns extend the built-in list", () => { + const out = redactAttributes({ tenant_ref: "t-1", other: "x" }, { + additionalKeyPatterns: [/tenant_ref/], + }); + assert.deepEqual(out, { tenant_ref: MASK, other: "x" }); +}); + +test("the browser relay scrubs message, stack, name and nested context", () => { + const payload = sanitizeBrowserPayload({ + version: 1, + service: "web", + environment: "production", + events: [{ + type: "exception", + timestamp: "2026-01-01T00:00:00.000Z", + message: "fetch failed for https://api.test/x?token=abc123secret", + stack: "Error: x\n at f (https://app.test/a.js:1:1) jane@example.com", + name: "/verify/eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpM", + context: { form: { password: "p@ss", plan: "pro" }, note: "sk-abcdefghijklmnopqrstuvwxyz" }, + }], + }); + const [event] = payload.events; + assert.equal(event.message, `fetch failed for https://api.test/x?token=${MASK}`); + assert.ok(!event.stack.includes("jane@example.com")); + assert.equal(event.name, `/verify/${MASK}`); + assert.deepEqual(event.context, { form: { password: MASK, plan: "pro" }, note: MASK }); +}); diff --git a/test-vectors/redaction.json b/test-vectors/redaction.json new file mode 100644 index 0000000..8bdd62a --- /dev/null +++ b/test-vectors/redaction.json @@ -0,0 +1,196 @@ +{ + "$comment": "Shared secret/PII scrubbing vectors. Every redactor (runtime-node redactText, runtime-browser scrubText, the relay, the otlp-ingester scrubText, and adapters/python redact_text) must map `input` to `expect`. Strings given as arrays are joined with no separator: realistic-looking vendor tokens are split into fragments so secret scanners (GitHub push protection) don't flag the fixture. `skip` lists implementations that deliberately don't cover a case.", + "text": [ + { + "name": "JWT inside an error message", + "input": "invalid session eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c for user", + "expect": "invalid session [redacted] for user" + }, + { + "name": "Authorization: Bearer header line", + "input": "request failed: Authorization: Bearer abcdefghijklmnopqrstuvwxyz1234567890", + "expect": "request failed: Authorization: [redacted]" + }, + { + "name": "bare bearer token", + "input": "sent Bearer abcdefghijklmnopqrstuvwxyz1234567890 upstream", + "expect": "sent [redacted] upstream" + }, + { + "name": "Basic auth credentials", + "input": "auth header Basic dXNlcjpwYXNzd29yZDEyMw== rejected", + "expect": "auth header [redacted] rejected" + }, + { + "name": "'basic' as an ordinary word survives", + "input": "requires basic authentication", + "expect": "requires basic authentication" + }, + { + "name": "Cookie header keeps no cookie values", + "input": "Cookie: sid=abc123; theme=dark", + "expect": "Cookie: [redacted]" + }, + { + "name": "Set-Cookie header", + "input": "Set-Cookie: session=s3cr3tvalue; Path=/; HttpOnly", + "expect": "Set-Cookie: [redacted]" + }, + { + "name": "headers serialised as JSON", + "input": "{\"authorization\":\"Bearer abcdefghijklmnopqrstuvwxyz\",\"cookie\":\"sid=abc; x=y\",\"host\":\"api.internal\"}", + "expect": "{\"authorization\":\"[redacted]\",\"cookie\":\"[redacted]\",\"host\":\"api.internal\"}" + }, + { + "name": "postgres connection string in ECONNREFUSED message", + "input": "connect ECONNREFUSED postgres://admin:hunter2@db.internal:5432/app", + "expect": "connect ECONNREFUSED postgres://[redacted]@db.internal:5432/app" + }, + { + "name": "mongodb+srv connection string", + "input": "MongoServerError at mongodb+srv://svc:Pa55word@cluster0.abcde.mongodb.net/prod", + "expect": "MongoServerError at mongodb+srv://[redacted]@cluster0.abcde.mongodb.net/prod" + }, + { + "name": "redis URL with empty username", + "input": "redis://:hunter2@cache.internal:6379/0 unreachable", + "expect": "redis://[redacted]@cache.internal:6379/0 unreachable" + }, + { + "name": "mysql URL whose password contains @", + "input": "mysql://root:p@ss@db:3306/shop", + "expect": "mysql://[redacted]@db:3306/shop" + }, + { + "name": "AWS access key id", + "input": "using AKIAIOSFODNN7EXAMPLE for s3", + "expect": "using [redacted] for s3" + }, + { + "name": "OpenAI project key (sk-proj-)", + "input": ["key ", "sk-proj-", "abcdefghijklmnopqrstuvwxyz0123456789", " rejected"], + "expect": "key [redacted] rejected" + }, + { + "name": "Anthropic key (sk-ant-)", + "input": ["x-api-key ", "sk-ant-api03-", "abcdefghijklmnopqrstuvwxyz0123"], + "expect": "x-api-key [redacted]" + }, + { + "name": "Stripe live secret key", + "input": ["stripe ", "sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc", " failed"], + "expect": "stripe [redacted] failed" + }, + { + "name": "GitHub classic and fine-grained tokens", + "skip": ["browser"], + "input": ["ghp_", "abcdefghijklmnopqrstuvwxyz1234567890", " and ", "github_pat_", "11ABCDEFG0abcdefghijklmnopqrstuvwxyz"], + "expect": "[redacted] and [redacted]" + }, + { + "name": "Slack bot token", + "input": ["xox", "b-123456789012-abcdefghijklmnopqrstuv", " end"], + "expect": "[redacted] end" + }, + { + "name": "Google API key (AIza)", + "input": ["maps ", "AIza", "SyA1234567890abcdefghijklmnopqrstu", " quota"], + "expect": "maps [redacted] quota" + }, + { + "name": "PEM private key block", + "skip": ["browser"], + "input": "cfg -----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAabc\n-----END RSA PRIVATE KEY----- loaded", + "expect": "cfg [redacted] loaded" + }, + { + "name": "token and password in a URL query string", + "input": "GET https://api.example.test/cb?token=abc123def&page=2&password=hunter2", + "expect": "GET https://api.example.test/cb?token=[redacted]&page=2&password=[redacted]" + }, + { + "name": "api_key and access_token query params", + "input": "/v1/data?api_key=k-12345&access_token=tok_98765&x=1", + "expect": "/v1/data?api_key=[redacted]&access_token=[redacted]&x=1" + }, + { + "name": "password assignment in free text and JSON", + "input": "login failed password=hunter2 body={\"password\":\"hunter 2\",\"user\":\"bob\"}", + "expect": "login failed password=[redacted] body={\"password\":\"[redacted]\",\"user\":\"bob\"}" + }, + { + "name": "client_secret and DB_PASSWORD env style", + "input": "client_secret: abcd1234 DB_PASSWORD=s3cret!", + "expect": "client_secret: [redacted] DB_PASSWORD=[redacted]" + }, + { + "name": "JSON parse error message is left alone", + "input": "Unexpected token '<', \" Date: Thu, 8 Oct 2026 00:01:34 +0530 Subject: [PATCH 02/15] feat(ingester,browser): coded-error grouping, request columns, log promotion Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 105 ++- docs/CONTINUOUS-DETECTION.md | 57 +- docs/EXTERNAL-SOURCES.md | 25 +- packages/otlp-ingester/README.md | 34 + packages/otlp-ingester/package.json | 2 +- packages/otlp-ingester/src/clickhouse.ts | 114 +++- .../otlp-ingester/src/coded-errors.test.ts | 627 ++++++++++++++++++ packages/otlp-ingester/src/config.ts | 7 + packages/otlp-ingester/src/context.ts | 4 +- packages/otlp-ingester/src/error-fields.ts | 107 +++ packages/otlp-ingester/src/fingerprint.ts | 55 ++ packages/otlp-ingester/src/log-promotion.ts | 100 +++ packages/otlp-ingester/src/logs.ts | 210 +++++- packages/otlp-ingester/src/migrations.ts | 54 +- .../otlp-ingester/src/normalize-browser.ts | 10 + packages/otlp-ingester/src/normalize-otlp.ts | 10 + .../otlp-ingester/src/server.logs.test.ts | 1 + .../otlp-ingester/src/server.sink.test.ts | 1 + packages/otlp-ingester/src/server.ts | 61 +- packages/otlp-ingester/src/sink.test.ts | 2 + packages/otlp-ingester/src/sink.ts | 43 +- packages/otlp-ingester/src/types.ts | 18 + packages/runtime-browser/README.md | 50 +- packages/runtime-browser/package.json | 6 +- packages/runtime-browser/src/index.ts | 92 ++- .../test/coded-errors.test.mjs | 120 ++++ 26 files changed, 1868 insertions(+), 47 deletions(-) create mode 100644 packages/otlp-ingester/src/coded-errors.test.ts create mode 100644 packages/otlp-ingester/src/error-fields.ts create mode 100644 packages/otlp-ingester/src/log-promotion.ts create mode 100644 packages/runtime-browser/test/coded-errors.test.mjs diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c038fb7..273e035 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -6,7 +6,8 @@ browser app ──tiny JSON──▶ customer's same-origin relay ──▶ /v1/browser ─┐ ├─▶ normalise → fingerprint server app ──OTLP/HTTP────────────────────────────────────▶ /v1/traces ──┤ - /v1/metrics ─┘ + /v1/metrics ─┤ + /v1/logs ────┘ │ ┌───────────────────────────┤ ▼ ▼ @@ -43,8 +44,12 @@ Two key scopes separate frontend and backend credentials: | --- | --- | --- | --- | | `runtime_error_occurrences` | MergeTree | `(org_id, repository_id, fingerprint, occurred_at)` | 14 d | | `runtime_spans` | MergeTree | `(org_id, repository_id, trace_id, started_at)` | 7 d | -| `runtime_logs` | ReplacingMergeTree | `(org_id, repository_id, occurred_at, event_id)` | 14 d | +| `runtime_logs` | ReplacingMergeTree | `(org_id, repository_id, occurred_at, event_id)` | `LOG_TTL_DAYS` (14 d) | +| `runtime_request_1m` | AggregatingMergeTree (fed by `runtime_request_1m_mv`) | `(org_id, repository_id, service, environment, route, method, bucket_at)` | 90 d | | `runtime_metrics_1m` | SummingMergeTree | `(org_id, repository_id, service, environment, release, route, bucket_at)` | 90 d | +| `runtime_latency_histograms` | ReplacingMergeTree | `(org_id, repository_id, service, environment, bucket_at, point_id)` | 90 d (`METRICS_TTL_DAYS`) | +| `runtime_memory_samples` | ReplacingMergeTree | `(org_id, repository_id, service, environment, instance_id, metric, observed_at)` | 14 d | +| `runtime_platform_events` | ReplacingMergeTree | `(org_id, repository_id, event_id)` | 30 d | | `runtime_llm_calls` | MergeTree | `(org_id, repository_id, started_at)` | 90 d | | `runtime_profile_samples` | MergeTree | `(org_id, repository_id, service, environment, release, observed_at, profile_id)` | 7 d | | `runtime_source_maps` | ReplacingMergeTree | `(org_id, repository_id, release, filename)` | 30 d | @@ -60,6 +65,51 @@ magnitude smaller than HTTP, spend analysis needs per-call granularity for failed calls), and SDKs send GenAI spans unsampled (the errors-are-100% rule applies to money too). +`runtime_latency_histograms` stores per-route OTLP duration histograms +(bucket bounds + counts) so latency percentiles can be computed from +unsampled metrics. `runtime_memory_samples` holds per-instance process +memory/GC gauges and `runtime_platform_events` ECS/Kubernetes OOM kills and +restarts (`POST /v1/platform-events`), which the consumer correlates with +memory incidents. + +### Wide events: `runtime_logs` and `runtime_request_1m` + +`runtime_logs` holds plain log records, operation summaries and (from +runtime-node 1.5.0) **request summaries** — one row per HTTP request or job, +always kept (no sampling; `LOG_TTL_DAYS` is the volume knob). Migration +0012 lifts the attributes every query filters on into columns: + +| Column | Source attribute | Notes | +| --- | --- | --- | +| `kind` | `autter.operation.kind` | `request` / `operation` (1.4.0 operation summaries without it → `operation`); `''` for plain logs | +| `request_id` | `autter.request.id` | bloom-filter skip index `idx_logs_request_id` — "everything for request X" | +| `route` | `http.route` | query-stripped, id-normalised (`/orders/:id`) | +| `status_code` | `http.response.status_code` | `0` when absent | +| `error_code` | `autter.error.code` | only when it matches `CODE_PATTERN` | +| `ai_cost_usd` / `ai_calls` | `autter.operation.ai` (`cost_usd`, `calls`) or the flattened `autter.operation.ai.*` keys | AI rollup per summary | + +The full event (context, steps, inline `autter.operation.logs`) stays in the +`attributes` JSON. `runtime_request_1m_mv` aggregates `kind = 'request'` rows +into `runtime_request_1m` per route/method/minute (`method` comes from the +`http.request.method` attribute). Read it with `sum(request_count)`, +`sum(failed_count)` (`outcome = 'failed'`), `sum(duration_sum_ms)` and +`quantilesMerge(0.5, 0.95)(duration_quantiles)`. The view only sees rows +inserted after it exists, and it counts every insert: a batch retried by an +exporter (after a 503 or a lost 2xx) is collapsed in `runtime_logs` by +`event_id` but counted twice in the rollup — route stats are approximate +under retries; exact per-request answers come from `runtime_logs`. + +**Log promotion.** Logger-only mode and `@autter/runtime-edge` have no span +to record an exception on; they send error log records with `exception.*` +and `autter.capture.mode = "log"`. `/v1/logs` promotes such records (severity +≥ error) to server occurrences — fingerprinted, written to +`runtime_error_occurrences`, counted in `runtime_metrics_1m` and forwarded to +the sink. A record is skipped when its trace id already produced an +occurrence in the same batch or, per a ClickHouse lookup, within ±60 s for +the same org/repo (the lookup ignores the candidates' own deterministic +ids, so a retried batch never dedupes against itself). A failed lookup +promotes anyway. Records without a trace id are always promoted. + `runtime_metrics_1m` is pre-aggregated per minute; readers must `SUM(...) GROUP BY` because SummingMergeTree collapses rows at merge time, eventually. Percentiles come from sampled spans at query time — the rollup @@ -87,13 +137,18 @@ stacks or routes in SQL: | Column | Derivation | Aggregation use | | --- | --- | --- | -| `fingerprint` | hash of source+service+type+normalised message+top frames+normalised route | the issue group key | +| `fingerprint` | `code-v1`: hash of service+error code; `message-v1`: hash of source+service+type+normalised message+top frames+normalised route (see [Fingerprinting](#fingerprinting)) | the issue group key | | `severity` | SDK-declared (`autter.severity`); `autter.unhandled` ⇒ `fatal` | errors vs warnings, alert thresholds | | `message_normalized` | ids/numbers/quoted strings templated out | "what is this group" label | | `route_normalized` | `/users/8812` → `/users/:id` | errors-by-endpoint, low-cardinality | | `top_frames` (Array) | top ≤5 normalised stack frames | "point of error" drill-down | | `first_frame` | `top_frames[1]` | single-column GROUP BY for hotspot files | | `method` | `http.request.method` | split GET vs POST failures | +| `fingerprint_scheme` | `code-v1` when grouped by a valid error code, else `message-v1` | tell coded issues from message-grouped ones | +| `error_code` | `autter.error.code` (validated) | errors-by-code | +| `error_why` / `error_fix` / `error_link` | `autter.error.why` / `.fix` / `.link` (scrubbed; ≤1000 / ≤1000 / ≤500 chars, http(s) links only) | declared cause/remedy for triage and RCA | +| `expected` (UInt8) | `autter.error.expected` | expected business failures — recorded, never paged | +| `request_id` | `autter.request.id` (exception spans inherit it from their server span; browser failures read `x-request-id`) | join to the request summary; bloom index `idx_occ_request_id` | Severity is deliberately **not** part of the fingerprint: the same defect reported as a warning in one code path and an error in another stays one @@ -112,6 +167,26 @@ backend) should treat columns as additive-only within a major version. ## Fingerprinting +Two schemes; `fingerprint_scheme` records which one produced a row. + +**`code-v1`** — when the occurrence carries a valid `autter.error.code` +(`^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$`, ≤80 chars): + +``` +sha256("code-v1" + "\0" + service + "\0" + code).hex()[:32] +``` + +Source, error type, message, stack and route are deliberately left out, so +the same code from the trace path, log promotion, the browser or an external +connector (Sentry, PostHog, Datadog, Loki, generic webhook — mapped by the +Autter backend) lands in **one issue**. Route and message stay as facets on +the occurrence rows. Test vector: `payments-api` + `billing.declined` → +`1692d304df5e4edbd3a0bbac5d7658bb`. Invalid codes (`ECONNRESET`, ids, free +text) are ignored and the error falls back to `message-v1`. + +**`message-v1`** — every uncoded error, unchanged byte-for-byte from +pre-1.5.0 so existing issues never regroup: + `sha256(source + service + error_type + normalised_message + top_5_frames + normalised_route)`, truncated to 32 hex chars. @@ -214,6 +289,12 @@ failures group into issues like any other error. } ``` +Declared-error context keys (runtime-browser ≥1.4.0) are lifted into the +occurrence fields above: `autter.error.code|why|fix|link|expected` (from coded +errors and `autterErrorFromResponse`) and `autter.request.id` (the +`x-request-id` header of a failed fetch/XHR). Each is validated on its own; a +malformed value is dropped without rejecting the payload. + Event types: `exception`, `unhandled_rejection`, `session_start`, and `track_event` (carries a `name`; counted into `runtime_metrics_1m` as `request_count` on the synthetic route `event:` — coarse usage @@ -253,12 +334,28 @@ When `AUTTER_SINK_URL` is set, each ingest batch POSTs: "statusCode": 500, "traceId": "...", "sessionId": "", - "occurredAt": "2026-07-21T11:22:00.123Z" + "occurredAt": "2026-07-21T11:22:00.123Z", + "method": "POST", + "fingerprintScheme": "code-v1", + "errorCode": "billing.declined", + "why": "The card issuer rejected the charge", + "fix": "Ask the customer for another card", + "link": "https://docs.example.com/payments#declined", + "expected": true, + "requestId": "req_3f9a…" } ] } ``` +1.5.0 additions to occurrences, all optional and omitted when absent (the +payload stays `version: 1`; older consumers ignore unknown keys): +`errorCode`, `why`, `fix`, `link`, `expected`, `requestId`, `traceId`, +`route`, `method`, `statusCode`, `fingerprintScheme` (`message-v1` | +`code-v1`). `traceId`/`route`/`method`/`statusCode` used to be sent as `null` +when unknown; they are now left out instead. Occurrences promoted from +`/v1/logs` arrive through the same payload. + Batches also carry `metrics` (1-minute usage rollup points) and `llmCalls` (per-call LLM usage — provider, model, tokens, `costUsd`, `costSource`, `userId`, `status`, `startedAt`) whenever the ingest produced them — same diff --git a/docs/CONTINUOUS-DETECTION.md b/docs/CONTINUOUS-DETECTION.md index 5410629..7b13c45 100644 --- a/docs/CONTINUOUS-DETECTION.md +++ b/docs/CONTINUOUS-DETECTION.md @@ -36,6 +36,58 @@ with `autter.outcome.status=error`, `autter.outcome.name=`, and ensure failed traces are retained by the exporter. Autter groups the event as an `OutcomeFailure`; no exception or log is required. +## Coded errors and expected failures + +Ingester 1.5.0 groups an error by its **code** when it carries a valid +`autter.error.code` (`^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$`, at most 80 +characters): one code in one service is one issue, whatever the message, +stack, route or source (server spans, logger-only and edge log records, +the browser, and external connectors all use the same `code-v1` +fingerprint). Uncoded errors keep message-based grouping unchanged. + +```ts +import { defineRuntimeErrors } from "@autter/runtime-node"; + +export const billingErrors = defineRuntimeErrors("billing", { + declined: { status: 402, message: "Payment declined", expected: true, + why: "The card issuer rejected the charge", fix: "Ask the customer for another card" }, +}); +throw billingErrors.declined(); // code "billing.declined" +``` + +Any error with a `code` property in that shape benefits without a rewrite: +`captureException` in the Node, edge and browser SDKs reads +`code/why/fix/link/expected` from the error. Other OTel languages set the +attributes on the `exception` event (or its span): + +| Attribute | Meaning | +| --- | --- | +| `autter.error.code` | Grouping key (invalid values are ignored) | +| `autter.error.why` / `autter.error.fix` | Declared cause and remedy, up to 1,000 characters each | +| `autter.error.link` | http(s) docs link, up to 500 characters | +| `autter.error.expected` | `true` for an expected business failure | +| `autter.request.id` | Request id; links the error to its request summary | + +The declared `why` and `fix` are shown on the issue and handed to root-cause +analysis **as declared**: the analysis confirms or contradicts them with +evidence rather than repeating them. + +**Expected failures** (`expected: true` — declines, validation, quota) are +still recorded and counted, but they never open incidents or trigger +automatic fixes. They are hidden from the default issues view; include them +with the "expected" filter. Use them for outcomes your product handles on +purpose, not to silence real defects. + +In the browser, `autterErrorFromResponse(res)` turns a failed `fetch` +response with an `{ "error": { "code", … } }` body into an error carrying +the server's code, and automatically observed 5xx responses carry their +`x-request-id` so the browser failure links to the server request. + +Services without a trace pipeline (logger-only mode and +`@autter/runtime-edge`) send errors as log records marked +`autter.capture.mode=log`; the ingester promotes those into issues, skipping +any whose trace already produced one in the last minute. + ## Traces and profiles Install framework, database, HTTP client, and queue instrumentations so a slow @@ -104,7 +156,10 @@ expanded detection or automatic drafts off in Runtime Overview. ## Rollout order Deploy the ingester first so it creates the profile and source map ClickHouse -tables (including migrations for existing databases). Then deploy the backend +tables (including migrations for existing databases). For coded errors and +request summaries, ingester 1.5.0 must be deployed before runtime-node/next +1.5.0, runtime-browser 1.4.0 or runtime-edge 1.0.0 send them; older ingesters +keep codes only inside the attributes JSON and group by message. Then deploy the backend and frontend; their organization tables are created for new organizations and upgraded on first use for existing ones. Finally publish the SDKs and enable profilers or the optional caught exception hooks service by service. Profiles diff --git a/docs/EXTERNAL-SOURCES.md b/docs/EXTERNAL-SOURCES.md index e0ab2af..b26ac74 100644 --- a/docs/EXTERNAL-SOURCES.md +++ b/docs/EXTERNAL-SOURCES.md @@ -6,7 +6,7 @@ The Autter platform can ingest repository-scoped logs from Sentry, PostHog, Graf This repository contains the SDKs, OTLP ingester and telemetry contracts. External provider polling and webhook collection run in the Autter platform's separate collector and analysis workers. Provider-only collection requires no application SDK upgrade, OTLP ingest key or ClickHouse schema migration. Keep your current SDK/OTel instrumentation if you also need application spans, source maps, usage or LLM telemetry. -SDK telemetry continues through the OTLP ingester and ClickHouse. External occurrences are normalized and stored in the platform's organization database, associated with the selected repository. Both feed Runtime issue investigation and the existing draft-fix executor. Deduplication handles provider event retries and polling/webhook overlap within a source; SDK/provider events are not guaranteed to merge into one group. +SDK telemetry continues through the OTLP ingester and ClickHouse. External occurrences are normalized and stored in the platform's organization database, associated with the selected repository. Both feed Runtime issue investigation and the existing draft-fix executor. Deduplication handles provider event retries and polling/webhook overlap within a source. Uncoded SDK and provider events are not guaranteed to merge into one group; **coded** errors are (see [Error codes](#error-codes)). ## Setup @@ -36,12 +36,33 @@ Send this shape to the private URL provided by your repository connection. Do no "occurredAt": "2026-09-28T12:00:00Z", "stack": "TypeError: Cannot read properties of undefined\n at checkout (src/checkout.ts:42:3)", "release": "deployed-commit-sha", - "traceId": "trace-id" + "traceId": "trace-id", + "errorCode": "billing.declined" } ``` +`errorCode` is optional. When it is a valid code (see below) the record is grouped by code rather than by `fingerprint`; `fingerprint` is still required and kept for provider links and dedupe. + Use `{"records":[...]}` for batches of up to 100. Alerts use `kind: "alert"`, `action: "fired"`; recovery uses the same fingerprint/service/environment with a new event ID, later timestamp and `action: "resolved"`. If configured, generic/Datadog webhooks also require the `x-autter-signature` shared-secret header; Sentry uses its integration HMAC. +## Error codes + +An error code is a namespaced, stable, low-cardinality identifier such as `billing.declined` or `auth.session_expired`. It must match `^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$` and be at most 80 characters. Codes never contain ids or user data. A value that doesn't match (`ECONNRESET`, `E_FAIL`, a UUID) is ignored and the record groups as before. + +A record with a valid code gets the source-independent `code-v1` fingerprint, `sha256("code-v1\0" + service + "\0" + code)[:32]` — the same one the OTLP ingester computes for SDK errors with `autter.error.code`. So one code in one service is **one issue** whether it came from the Node/browser/edge SDKs, Sentry, PostHog, Datadog, Loki or a generic webhook; the issue lists every source that reported it and keeps each provider's own issue id for links. Records without a valid code keep the provider fingerprint. + +The platform reads the code from the first field that holds one: + +| Provider | Where the code is read from (first match wins) | +| --- | --- | +| Sentry | `tags.error_code`, `tags.code`, `contexts.autter.error_code`, `exception.values[-1].mechanism.data.code`, `extra.code` | +| PostHog | `properties.error_code`, `properties.$exception_list[-1].code`, `properties.autter_error_code` | +| Datadog | `@error.code`, `attributes.error.code`, tag `error_code:` | +| Grafana/Loki | Stream label `error_code`, or `code` / `error.code` in a JSON log line | +| Generic webhook | `errorCode` | + +To use this, set the code where your provider SDK already lets you attach data — for example a Sentry tag `error_code`, or a structured log field `error.code` for Loki and Datadog. Above 500 distinct codes per service per day, new codes fall back to message grouping and Runtime shows a "code cardinality too high" notice; this guards against ids leaking into codes. + Do not forward secrets, cookies, request bodies or personal data unnecessarily. The platform bounds and redacts stored diagnostic fields; upstream scrubbing is still recommended. Provider credentials, private webhook URLs, Runtime server/client ingest keys and CLI personal tokens serve different purposes. Self-hosted platform operators must deploy the API, collector, analysis worker and existing PR executor with their database/GitHub/AI configuration. API and collectors need the same persistent connector encryption key. These workers are not included in the standalone OTLP ingester image. diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 1f3a360..c485671 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -9,6 +9,7 @@ one per-repo signal model, fingerprints errors, and writes ClickHouse. | Route | Payload | Purpose | | --- | --- | --- | | `POST /v1/traces` | OTLP/JSON `ExportTraceServiceRequest` | Error spans → occurrences; all spans → `runtime_spans`; server spans → usage rollups; GenAI spans → `runtime_llm_calls` | +| `POST /v1/logs` | OTLP `ExportLogsServiceRequest`, server key | Wide events (request/operation summaries, logs) → `runtime_logs` (+ `runtime_request_1m` rollup); logger-only error records (`autter.capture.mode=log`) → occurrences | | `POST /v1/metrics` | OTLP `ExportMetricsServiceRequest` | HTTP-server duration histograms → usage rollups; portable process memory/GC metrics → per-instance memory samples | | `POST /v1/platform-events` | JSON, server key | ECS/Kubernetes OOM kills and restarts → memory incident correlation | | `POST /v1/profiles` | Symbolized pprof, server key | CPU/in-use heap profile samples | @@ -66,6 +67,39 @@ The validator webhook may return the same extra fields: | `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) | | `OCCURRENCE_TTL_DAYS` / `SPAN_TTL_DAYS` / `METRICS_TTL_DAYS` | `14` / `7` / `90` | ClickHouse TTLs (applied at table creation) | | `LLM_CALL_TTL_DAYS` | `90` | Retention for `runtime_llm_calls` rows | +| `LOG_TTL_DAYS` | `14` | Retention for `runtime_logs` (request summaries are always kept, never sampled, so this is the main volume knob). Unlike the other TTLs it is also applied to **existing** tables: at boot the ingester compares it with the table's TTL and runs `ALTER TABLE … MODIFY TTL` only when they differ; a failure is logged and retried next boot | + +## Coded errors and request summaries (1.5.0) + +- **Code-based grouping.** An occurrence carrying a valid `autter.error.code` + (`^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$`, ≤80 chars) is fingerprinted by + scheme `code-v1`: `sha256("code-v1\0" + service + "\0" + code)[:32]` — + independent of source, message, stack and route, so one code is one issue + across traces, log promotion, the browser and external connectors. + Uncoded errors keep the historical fingerprint (`message-v1`) byte-for-byte. + `fingerprint_scheme` is stored per occurrence and sent to the sink. +- **Declared fields.** `autter.error.why|fix|link|expected` and + `autter.request.id` are lifted from span/exception-event attributes, log + attributes and browser context into `runtime_error_occurrences` + (`error_why`, `error_fix`, `error_link`, `expected`, `request_id`) and the + sink payload. why/fix are scrubbed and capped at 1000 chars; links must be + http(s) and ≤500 chars. Exception spans inherit the request id of their + server span. +- **Request summaries.** `/v1/logs` lifts `autter.operation.kind`, + `autter.request.id`, `http.route` (id-normalised), `http.response.status_code`, + `autter.error.code` and the `autter.operation.ai` rollup (`cost_usd`, `calls`) + into `runtime_logs` columns. `kind = 'request'` rows feed the + `runtime_request_1m` materialized view (per route/method/minute: count, + failed, duration sum, p50/p95). The view counts every insert, so a retried + batch can be counted twice — route stats are approximate under exporter + retries; `runtime_logs` itself collapses duplicates. +- **Log promotion.** SDKs without a trace pipeline (logger-only mode, + `@autter/runtime-edge`) send errors as log records with `exception.*` and + `autter.capture.mode = "log"`. `/v1/logs` promotes those (severity ≥ error) + to server occurrences — stored, counted and forwarded to the sink — + skipping a record when its trace id already produced an occurrence in the + same batch or (ClickHouse lookup) within ±60 s. If the lookup fails the + record is promoted anyway. ## LLM / GenAI calls diff --git a/packages/otlp-ingester/package.json b/packages/otlp-ingester/package.json index 24aa3a3..ed631e1 100644 --- a/packages/otlp-ingester/package.json +++ b/packages/otlp-ingester/package.json @@ -1,6 +1,6 @@ { "name": "@autter/otlp-ingester", - "version": "1.4.0", + "version": "1.5.0", "description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model", "license": "MIT", "type": "module", diff --git a/packages/otlp-ingester/src/clickhouse.ts b/packages/otlp-ingester/src/clickhouse.ts index 2a37371..ea4ba91 100644 --- a/packages/otlp-ingester/src/clickhouse.ts +++ b/packages/otlp-ingester/src/clickhouse.ts @@ -8,9 +8,15 @@ import { latencyTableDDL, type LatencyHistogram } from "./latency.js"; import { profileTableDDL, type ProfileSample } from "./profiles.js"; import { memoryTableDDL, platformEventTableDDL, type MemorySample, type PlatformEvent } from "./memory.js"; import { sourceMapTableDDL } from "./source-maps.js"; -import { logTableDDL, type RuntimeLogRecord } from "./logs.js"; +import { + logTableDDL, + requestRollupTableDDL, + type RuntimeLogRecord, +} from "./logs.js"; import { MIGRATIONS, + OCCURRENCE_CODE_COLUMNS, + OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, type Migration, } from "./migrations.js"; @@ -35,6 +41,11 @@ const INSERT_SETTINGS: ClickHouseSettings = { wait_for_async_insert: 1, }; +/** "YYYY-MM-DD hh:mm:ss.sss" (UTC) — the DateTime64 query-parameter form. */ +function chDateTime(date: Date): string { + return date.toISOString().replace("T", " ").replace("Z", ""); +} + export class ClickHouseStore { private client: ClickHouseClient | null = null; private ensurePromise: Promise | null = null; @@ -69,6 +80,7 @@ export class ClickHouseStore { const db = this.config.clickhouseDatabase; const { occurrenceTtlDays, spanTtlDays, metricsTtlDays, llmCallTtlDays } = this.config; + const logTtlDays = this.config.logTtlDays ?? 14; return [ `CREATE DATABASE IF NOT EXISTS ${db}`, latencyTableDDL(db, metricsTtlDays), @@ -76,7 +88,11 @@ export class ClickHouseStore { memoryTableDDL(db), platformEventTableDDL(db), sourceMapTableDDL(db), - logTableDDL(db), + logTableDDL(db, logTtlDays), + // The table only: its materialized view reads 0012 columns that an + // existing runtime_logs lacks until migrations run, so the view is + // created by migration 0014 alone (migrations also run on fresh DBs). + requestRollupTableDDL(db), `CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences ( org_id String, repository_id String, @@ -101,7 +117,9 @@ export class ClickHouseStore { session_id String DEFAULT '', attributes String DEFAULT '{}' CODEC(ZSTD(1)), occurred_at DateTime64(3, 'UTC'), - ingested_at DateTime64(3, 'UTC') DEFAULT now64(3) + ingested_at DateTime64(3, 'UTC') DEFAULT now64(3), + ${OCCURRENCE_CODE_COLUMNS.join(",\n\t\t\t\t")}, + ${OCCURRENCE_REQUEST_ID_INDEX} ) ENGINE = MergeTree PARTITION BY toDate(occurred_at) @@ -217,6 +235,7 @@ export class ClickHouseStore { }); console.log(`clickhouse migration applied: ${migration.id}`); } + await this.applyLogTtl(); })().catch((err) => { this.ensurePromise = null; throw err; @@ -224,6 +243,84 @@ export class ClickHouseStore { return this.ensurePromise; } + /** + * Bring runtime_logs' TTL in line with LOG_TTL_DAYS. Idempotent: reads the + * current TTL from system.tables and only issues `MODIFY TTL` when it + * differs (MODIFY TTL rewrites TTL info for existing parts, so it must not + * run on every boot). Best-effort — a failure is logged and never blocks + * ingest; the next boot retries. + */ + async applyLogTtl(): Promise { + const days = this.config.logTtlDays ?? 14; + const db = this.config.clickhouseDatabase; + try { + const client = this.getClient(); + const rows = await client.query({ + query: `SELECT engine_full FROM system.tables WHERE database = {db:String} AND name = 'runtime_logs'`, + query_params: { db }, + format: "JSONEachRow", + }); + const [table] = await rows.json<{ engine_full: string }>(); + if (!table) return; + const current = /\bTTL\b[^]*?toIntervalDay\((\d+)\)/.exec(table.engine_full); + if (!current) { + console.warn( + "runtime_logs TTL not recognised — LOG_TTL_DAYS not applied", + ); + return; + } + if (Number(current[1]) === days) return; + await client.command({ + query: `ALTER TABLE ${db}.runtime_logs MODIFY TTL toDateTime(occurred_at) + INTERVAL ${days} DAY`, + }); + console.log( + `runtime_logs TTL changed from ${current[1]} to ${days} days (LOG_TTL_DAYS)`, + ); + } catch (err) { + console.warn( + "applying LOG_TTL_DAYS to runtime_logs failed (will retry next boot):", + err instanceof Error ? err.message : err, + ); + } + } + + /** + * Trace ids (from `traceIds`) that already produced an occurrence for + * this tenant in [from, to] — the cross-batch half of the log-promotion + * dedupe. `excludeIds` are the candidate occurrences' own deterministic + * ids, so a retried batch does not dedupe against its own earlier, + * partially-committed write. Throws on failure; the caller decides. + */ + async recentOccurrenceTraceIds( + ctx: IngestContext, + traceIds: string[], + from: Date, + to: Date, + excludeIds: string[], + ): Promise> { + if (traceIds.length === 0 || !this.configured) return new Set(); + await this.ensureSchema(); + const rows = await this.getClient().query({ + query: `SELECT DISTINCT trace_id FROM ${this.table("runtime_error_occurrences")} + WHERE org_id = {org:String} AND repository_id = {repo:String} + AND occurred_at >= {from:DateTime64(3, 'UTC')} AND occurred_at <= {to:DateTime64(3, 'UTC')} + AND has({traces:Array(String)}, trace_id) + AND NOT has({exclude:Array(String)}, occurrence_id)`, + query_params: { + org: ctx.orgId, + repo: ctx.repositoryId, + from: chDateTime(from), + to: chDateTime(to), + traces: traceIds, + exclude: excludeIds, + }, + format: "JSONEachRow", + }); + return new Set( + (await rows.json<{ trace_id: string }>()).map((row) => row.trace_id), + ); + } + async ping(): Promise { if (!this.configured) return false; const result = await this.getClient().ping(); @@ -264,6 +361,13 @@ export class ClickHouseStore { session_id: o.sessionId ?? "", attributes: JSON.stringify(o.attributes ?? {}), occurred_at: o.occurredAt.toISOString(), + error_code: o.errorCode ?? "", + error_why: o.why ?? "", + error_fix: o.fix ?? "", + error_link: o.link ?? "", + expected: o.expected ? 1 : 0, + request_id: o.requestId ?? "", + fingerprint_scheme: o.fingerprintScheme, })), }); } @@ -305,7 +409,9 @@ export class ClickHouseStore { service: row.service, environment: row.environment, release: row.release, trace_id: row.traceId, span_id: row.spanId, operation_id: row.operationId, operation: row.operation, event_type: row.type, severity: row.severity, message: row.message, outcome: row.outcome, duration_ms: row.durationMs, - attributes: JSON.stringify(row.attributes), occurred_at: row.occurredAt.toISOString() })) }); + attributes: JSON.stringify(row.attributes), occurred_at: row.occurredAt.toISOString(), + kind: row.kind, request_id: row.requestId, route: row.route, status_code: row.statusCode, + error_code: row.errorCode, ai_cost_usd: row.aiCostUsd, ai_calls: row.aiCalls })) }); } async insertProfileSamples(ctx: IngestContext, samples: ProfileSample[]): Promise { diff --git a/packages/otlp-ingester/src/coded-errors.test.ts b/packages/otlp-ingester/src/coded-errors.test.ts new file mode 100644 index 0000000..9e0d40d --- /dev/null +++ b/packages/otlp-ingester/src/coded-errors.test.ts @@ -0,0 +1,627 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { test } from "node:test"; +import { ClickHouseStore } from "./clickhouse.js"; +import type { IngesterConfig } from "./config.js"; +import { liftErrorFields } from "./error-fields.js"; +import { + codeFingerprint, + fingerprintOccurrence, + normalizeMessage, + normalizeRoute, + normalizeStackFrames, + occurrenceFingerprint, + validErrorCode, +} from "./fingerprint.js"; +import { logPromotionCandidates } from "./log-promotion.js"; +import { logTableDDL, normalizeLogs, requestRollupViewDDL } from "./logs.js"; +import { MIGRATIONS } from "./migrations.js"; +import { normalizeBrowserPayload } from "./normalize-browser.js"; +import { normalizeTraces } from "./normalize-otlp.js"; +import { createIngesterApp } from "./server.js"; +import { sinkOccurrence } from "./sink.js"; +import type { RuntimeOccurrence, RuntimeOccurrenceInput } from "./types.js"; + +const str = (key: string, value: string) => ({ key, value: { stringValue: value } }); +const int = (key: string, value: number) => ({ key, value: { intValue: value } }); +const bool = (key: string, value: boolean) => ({ key, value: { boolValue: value } }); + +// ── code-v1 fingerprint (shared vectors with the backend) ───────────────── + +const CODE_VECTORS = [ + { service: "payments-api", code: "billing.declined", fingerprint: "1692d304df5e4edbd3a0bbac5d7658bb" }, + { service: "payments-api", code: "billing.limit", fingerprint: "be8302686d461915fb67bd28cdba26ba" }, + { service: "web-app", code: "auth.session_expired", fingerprint: "b997fe46c7a61540917324f78b710d83" }, + { service: "autter-api", code: "github.app_permission_missing", fingerprint: "ce80dc4c785f51a0851e99cd5dd1316f" }, +]; + +function input(overrides: Partial = {}): RuntimeOccurrenceInput { + return { + source: "server", + severity: "error", + service: "payments-api", + environment: "prod", + release: null, + errorType: "RuntimeError", + message: "Payment declined for order 42", + stack: "Error: x\n at charge (/app/billing.ts:10:3)", + route: "/orders/42/pay", + method: "POST", + statusCode: 402, + traceId: "a".repeat(32), + sessionId: null, + attributes: null, + occurredAt: new Date("2026-01-01T00:00:00.000Z"), + ...overrides, + }; +} + +test("code-v1 fingerprints match the shared test vectors", () => { + for (const vector of CODE_VECTORS) { + assert.equal(codeFingerprint(vector.service, vector.code), vector.fingerprint); + } +}); + +test("a valid code groups source-independently; uncoded errors keep message-v1 byte-for-byte", () => { + const server = occurrenceFingerprint(input({ errorCode: "billing.declined" })); + assert.deepEqual(server, { fingerprint: "1692d304df5e4edbd3a0bbac5d7658bb", scheme: "code-v1" }); + // Different source, message, stack and route: still the same issue. + const browser = occurrenceFingerprint( + input({ errorCode: "billing.declined", source: "browser", message: "other", stack: null, route: "/x" }), + ); + assert.equal(browser.fingerprint, server.fingerprint); + + const legacy = input(); + const expected = createHash("sha256") + .update( + [ + legacy.source, + legacy.service, + legacy.errorType, + normalizeMessage(legacy.message), + ...normalizeStackFrames(legacy.stack), + normalizeRoute(legacy.route), + ].join(" "), + ) + .digest("hex") + .slice(0, 32); + assert.deepEqual(occurrenceFingerprint(legacy), { fingerprint: expected, scheme: "message-v1" }); + // An invalid code falls back to the message fingerprint. + assert.equal(fingerprintOccurrence(input({ errorCode: "ECONNRESET" })), expected); +}); + +test("validErrorCode enforces CODE_PATTERN and the 80-char cap", () => { + for (const ok of ["billing.declined", "a", "a.b.c.d", "inventory.reservation_timeout", "x1.y_2"]) { + assert.equal(validErrorCode(ok), ok); + } + for (const bad of ["ECONNRESET", "Billing.declined", "1abc", "a.b.c.d.e", "a..b", "a.", "a-b", "user.42@x", "", 42, null, "a" + "b".repeat(80)]) { + assert.equal(validErrorCode(bad), undefined, String(bad)); + } +}); + +// ── declared error fields ───────────────────────────────────────────────── + +test("declared fields are validated, scrubbed and capped independently", () => { + const fields = liftErrorFields( + new Map([ + ["autter.error.code", "billing.declined"], + ["autter.error.why", "x".repeat(1500)], + ["autter.error.fix", "Email jane@example.com for help"], + ["autter.error.link", "https://user:pw@docs.example.com/payments?token=1#declined"], + ["autter.error.expected", "true"], + ["autter.request.id", "req_0123456789"], + ]), + ); + assert.equal(fields.errorCode, "billing.declined"); + assert.equal(fields.why!.length, 1000); + assert.equal(fields.fix, "Email [redacted] for help"); + assert.equal(fields.link, "https://docs.example.com/payments#declined"); + assert.equal(fields.expected, true); + assert.equal(fields.requestId, "req_0123456789"); + + const rejected = liftErrorFields({ + "autter.error.code": "Not A Code", + "autter.error.link": "javascript:alert(1)", + "autter.error.expected": "yes", + "autter.request.id": "short", + }); + assert.deepEqual(rejected, {}); + assert.equal(liftErrorFields({ "autter.error.link": `https://x.dev/${"a".repeat(600)}` }).link, undefined); + // First source wins (exception event before its span). + assert.equal( + liftErrorFields({ "autter.error.code": "a.event" }, { "autter.error.code": "a.span" }).errorCode, + "a.event", + ); +}); + +test("OTLP exception events lift codes, and child error spans inherit the request id", () => { + const traceId = "c".repeat(32); + const { occurrences } = normalizeTraces({ + resourceSpans: [ + { + resource: { attributes: [str("service.name", "payments-api")] }, + scopeSpans: [ + { + spans: [ + { + traceId, + spanId: "1".repeat(16), + name: "POST /pay", + kind: 2, + startTimeUnixNano: "1760000000000000000", + endTimeUnixNano: "1760000000100000000", + attributes: [ + str("http.route", "/pay"), + str("http.request.method", "POST"), + int("http.response.status_code", 402), + str("autter.request.id", "req_abcdefgh12"), + ], + }, + { + traceId, + spanId: "2".repeat(16), + parentSpanId: "1".repeat(16), + name: "Error", + startTimeUnixNano: "1760000000010000000", + endTimeUnixNano: "1760000000020000000", + attributes: [str("autter.error.code", "billing.span_level")], + events: [ + { + name: "exception", + timeUnixNano: "1760000000015000000", + attributes: [ + str("exception.type", "RuntimeError"), + str("exception.message", "Payment declined"), + str("autter.error.code", "billing.declined"), + str("autter.error.why", "The issuer rejected the charge"), + str("autter.error.fix", "Ask for another card"), + str("autter.error.link", "https://docs.example.com/payments#declined"), + bool("autter.error.expected", true), + ], + }, + ], + }, + ], + }, + ], + }, + ], + }); + assert.equal(occurrences.length, 1); + const [occ] = occurrences; + assert.equal(occ!.errorCode, "billing.declined"); + assert.equal(occ!.why, "The issuer rejected the charge"); + assert.equal(occ!.fix, "Ask for another card"); + assert.equal(occ!.link, "https://docs.example.com/payments#declined"); + assert.equal(occ!.expected, true); + assert.equal(occ!.requestId, "req_abcdefgh12"); + assert.equal(occ!.route, "/pay"); +}); + +test("browser context keys become occurrence fields; context sanitiser keeps them", () => { + const { occurrences } = normalizeBrowserPayload({ + version: 1, + service: "web-app", + environment: "prod", + events: [ + { + type: "exception", + timestamp: "2026-01-01T00:00:00.000Z", + message: "Session expired", + context: { + "autter.error.code": "auth.session_expired", + "autter.error.why": "Token older than 24h", + "autter.error.fix": "Sign in again", + "autter.error.link": "https://docs.example.com/auth#expired", + "autter.error.expected": true, + }, + }, + { + type: "request_failure", + timestamp: "2026-01-01T00:00:01.000Z", + name: "/api/checkout", + message: "Request returned 503", + errorType: "HttpRequestError", + context: { "autter.request.id": "0f6c1d2e-aaaa-bbbb-cccc-000000000001", "autter.error.code": "BAD CODE" }, + }, + ], + }); + const [coded, failure] = occurrences; + assert.equal(coded!.errorCode, "auth.session_expired"); + assert.equal(coded!.why, "Token older than 24h"); + assert.equal(coded!.fix, "Sign in again"); + assert.equal(coded!.link, "https://docs.example.com/auth#expired"); + assert.equal(coded!.expected, true); + const context = coded!.attributes!.context as Record; + assert.equal(context["autter.error.code"], "auth.session_expired"); + assert.equal(context["autter.error.why"], "Token older than 24h"); + assert.equal(failure!.requestId, "0f6c1d2e-aaaa-bbbb-cccc-000000000001"); + assert.equal(failure!.errorCode, undefined); + assert.equal(occurrenceFingerprint(coded!).fingerprint, "b997fe46c7a61540917324f78b710d83"); +}); + +// ── runtime_logs lifting ────────────────────────────────────────────────── + +test("request summaries lift kind, request id, route, status, code and AI rollup", () => { + const [summary, legacyOp, plain] = normalizeLogs({ + resourceLogs: [ + { + resource: { attributes: [str("service.name", "api")] }, + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1760000000000000000", + body: { stringValue: "POST /orders/:id/pay" }, + attributes: [ + str("autter.event.type", "operation"), + str("autter.operation.kind", "request"), + str("autter.request.id", "req_abcdefgh12"), + str("http.route", "/orders/123/pay?coupon=x"), + str("http.request.method", "POST"), + int("http.response.status_code", 402), + str("autter.error.code", "billing.declined"), + str("autter.operation.ai", JSON.stringify({ calls: 3, cost_usd: 0.0125, cache_read_tokens: 10 })), + ], + }, + { + timeUnixNano: "1760000000000000001", + body: { stringValue: "sync" }, + attributes: [ + str("autter.event.type", "operation"), + str("autter.error.code", "NOT_VALID"), + str("autter.operation.ai.cost_usd", "0.5"), + int("autter.operation.ai.calls", 2), + ], + }, + { timeUnixNano: "1760000000000000002", body: { stringValue: "hello" } }, + ], + }, + ], + }, + ], + }); + assert.equal(summary!.kind, "request"); + assert.equal(summary!.requestId, "req_abcdefgh12"); + assert.equal(summary!.route, "/orders/:id/pay"); + assert.equal(summary!.statusCode, 402); + assert.equal(summary!.errorCode, "billing.declined"); + assert.equal(summary!.aiCalls, 3); + assert.equal(summary!.aiCostUsd, 0.0125); + assert.equal( + (summary!.attributes["autter.operation.ai"] as Record).cache_read_tokens, + 10, + ); + assert.equal(summary!.occurrence, undefined); + // 1.4.0 operation summaries (no kind) are "operation"; invalid codes drop. + assert.equal(legacyOp!.kind, "operation"); + assert.equal(legacyOp!.errorCode, ""); + assert.equal(legacyOp!.aiCostUsd, 0.5); + assert.equal(legacyOp!.aiCalls, 2); + assert.equal(plain!.kind, ""); + assert.equal(plain!.route, ""); + assert.equal(plain!.statusCode, 0); +}); + +test("only logger-only error records with exception.* become promotion candidates", () => { + const record = (attrs: ReturnType[], severityNumber = 17, traceId?: string) => ({ + timeUnixNano: "1760000000000000000", + severityNumber, + ...(traceId ? { traceId } : {}), + body: { stringValue: "failed" }, + attributes: attrs, + }); + const mode = str("autter.capture.mode", "log"); + const exc = str("exception.message", "Payment declined"); + const rows = normalizeLogs({ + resourceLogs: [ + { + resource: { attributes: [str("service.name", "edge-api")] }, + scopeLogs: [ + { + logRecords: [ + record([mode, exc, str("exception.type", "RuntimeError"), str("autter.error.code", "billing.declined"), + str("autter.error.link", "https://docs.example.com/p#declined"), str("http.route", "/pay")], 17, "d".repeat(32)), + record([mode, exc], 17, "d".repeat(32)), // same trace, same batch + record([mode, exc], 13), // warning + record([exc]), // no capture mode + record([mode]), // no exception + record([mode, exc], 21), // fatal, no trace + ], + }, + ], + }, + ], + }); + assert.deepEqual(rows.map((r) => Boolean(r.occurrence)), [true, true, false, false, false, true]); + const first = rows[0]!.occurrence!; + assert.equal(first.errorCode, "billing.declined"); + assert.equal(first.link, "https://docs.example.com/p#declined"); + assert.equal(first.errorType, "RuntimeError"); + assert.equal(first.route, "/pay"); + assert.equal(rows[0]!.errorCode, "billing.declined"); + const candidates = logPromotionCandidates(rows); + assert.equal(candidates.length, 2); + assert.equal(candidates[1]!.severity, "fatal"); +}); + +// ── sink payload ────────────────────────────────────────────────────────── + +test("sink occurrences carry the 1.5.0 optional fields and omit absent ones", () => { + const base = { + ...input({ errorCode: "billing.declined", why: "w", fix: "f", link: "https://d.dev/#x", expected: true, requestId: "req_abcdefgh12" }), + occurrenceId: "occ-1", + fingerprint: "fp", + fingerprintScheme: "code-v1", + routeNormalized: "/orders/:id/pay", + messageNormalized: "m", + topFrames: [], + firstFrame: "", + } satisfies RuntimeOccurrence; + const wire = JSON.parse(JSON.stringify(sinkOccurrence(base))); + assert.equal(wire.errorCode, "billing.declined"); + assert.equal(wire.why, "w"); + assert.equal(wire.fix, "f"); + assert.equal(wire.link, "https://d.dev/#x"); + assert.equal(wire.expected, true); + assert.equal(wire.requestId, "req_abcdefgh12"); + assert.equal(wire.traceId, "a".repeat(32)); + assert.equal(wire.route, "/orders/42/pay"); + assert.equal(wire.method, "POST"); + assert.equal(wire.statusCode, 402); + assert.equal(wire.fingerprintScheme, "code-v1"); + assert.equal(wire.occurredAt, "2026-01-01T00:00:00.000Z"); + + const bare = JSON.parse( + JSON.stringify( + sinkOccurrence({ ...base, errorCode: undefined, why: undefined, fix: undefined, link: undefined, + expected: undefined, requestId: undefined, traceId: null, route: null, method: null, statusCode: null, + fingerprintScheme: "message-v1" }), + ), + ); + for (const key of ["errorCode", "why", "fix", "link", "expected", "requestId", "traceId", "route", "method", "statusCode"]) { + assert.equal(key in bare, false, key); + } + assert.equal(bare.fingerprintScheme, "message-v1"); +}); + +// ── migrations ──────────────────────────────────────────────────────────── + +test("migrations are uniquely ordered and 0012–0014 are additive and idempotent", () => { + const ids = MIGRATIONS.map((m) => m.id); + assert.equal(new Set(ids).size, ids.length); + const serials = ids.map((id) => Number(id.slice(0, 4))); + assert.deepEqual(serials, serials.map((_, i) => i + 1)); + assert.deepEqual(ids.slice(-3), ["0012-runtime-logs-requests", "0013-occurrence-codes", "0014-runtime-request-1m"]); + for (const migration of MIGRATIONS.slice(-3)) { + for (const statement of migration.statements) { + assert.match(statement, /IF NOT EXISTS/, statement); + assert.doesNotMatch(statement, /DROP|MODIFY/, statement); + } + } + const [logs, occ, rollup] = MIGRATIONS.slice(-3).map((m) => m.statements.join("\n")); + for (const column of ["kind LowCardinality(String) DEFAULT ''", "request_id String DEFAULT ''", "route String DEFAULT ''", + "status_code UInt16 DEFAULT 0", "error_code String DEFAULT ''", "ai_cost_usd Float64 DEFAULT 0", "ai_calls UInt32 DEFAULT 0", + "INDEX IF NOT EXISTS idx_logs_request_id request_id TYPE bloom_filter GRANULARITY 4"]) { + assert.ok(logs!.includes(column), column); + } + for (const column of ["error_code String DEFAULT ''", "error_why String DEFAULT ''", "error_fix String DEFAULT ''", + "error_link String DEFAULT ''", "expected UInt8 DEFAULT 0", "request_id String DEFAULT ''", + "fingerprint_scheme LowCardinality(String) DEFAULT 'message-v1'", + "INDEX IF NOT EXISTS idx_occ_request_id request_id TYPE bloom_filter GRANULARITY 4"]) { + assert.ok(occ!.includes(column), column); + } + assert.match(rollup!, /CREATE TABLE IF NOT EXISTS \{db\}\.runtime_request_1m \(/); + assert.match(rollup!, /AggregatingMergeTree/); + assert.match(rollup!, /ORDER BY \(org_id, repository_id, service, environment, route, method, bucket_at\)/); + assert.match(rollup!, /TTL bucket_at \+ INTERVAL 90 DAY/); + assert.match(rollup!, /duration_quantiles AggregateFunction\(quantiles\(0\.5, 0\.95\), Float64\)/); + assert.match(rollup!, /CREATE MATERIALIZED VIEW IF NOT EXISTS \{db\}\.runtime_request_1m_mv\s+TO \{db\}\.runtime_request_1m/); + assert.match(requestRollupViewDDL("db"), /WHERE kind = 'request'/); + assert.match(requestRollupViewDDL("db"), /countIf\(outcome = 'failed'\) AS failed_count/); + // Fresh databases get the same runtime_logs shape and the configured TTL. + const baseline = logTableDDL("db", 30); + assert.match(baseline, /ai_calls UInt32 DEFAULT 0/); + assert.match(baseline, /INDEX idx_logs_request_id request_id TYPE bloom_filter GRANULARITY 4/); + assert.match(baseline, /INTERVAL 30 DAY/); + assert.match(logTableDDL("db"), /INTERVAL 14 DAY/); +}); + +// ── ClickHouse-backed behaviour (HTTP stub) ─────────────────────────────── + +interface StubCall { sql: string; params: URLSearchParams; body: string } + +async function startStub( + respond: (call: StubCall) => { status?: number; body?: string } | void, +): Promise<{ server: Server; url: string; calls: StubCall[] }> { + const calls: StubCall[] = []; + const server = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + const params = new URL(req.url!, "http://localhost").searchParams; + const call = { sql: `${params.get("query") ?? ""}\n${body}`, params, body }; + calls.push(call); + const out = respond(call) ?? {}; + res.writeHead(out.status ?? 200).end(out.body ?? ""); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return { server, url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, calls }; +} + +const close = (server: Server) => + new Promise((resolve) => { + server.close(() => resolve()); + server.closeAllConnections(); + }); + +function config(overrides: Partial): IngesterConfig { + return { + port: 0, + clickhouseUrl: null, + clickhouseUser: "default", + clickhousePassword: "", + clickhouseDatabase: "autter_runtime", + ingestKeys: [{ key: "server", orgId: "org-1", repositoryId: "repo-1" }], + keyValidatorUrl: null, + keyValidatorToken: null, + sinkUrl: null, + sinkToken: null, + sinkMaxAttempts: 1, + sinkMaxBufferedBatches: 10, + sinkMaxBufferedMb: 2, + maxBodyBytes: 1024 * 1024, + rateLimitPerMinute: 300, + clientRateLimitPerMinute: 120, + occurrenceTtlDays: 14, + spanTtlDays: 7, + metricsTtlDays: 90, + llmCallTtlDays: 90, + logTtlDays: 14, + ...overrides, + }; +} + +test("LOG_TTL_DAYS is applied only when it differs, and a failure never blocks the schema", async () => { + let engine = "ReplacingMergeTree PARTITION BY toDate(occurred_at) ORDER BY (org_id) TTL toDateTime(occurred_at) + toIntervalDay(14) SETTINGS index_granularity = 8192"; + let failTtl = false; + const stub = await startStub((call) => { + if (call.sql.includes("system.tables")) { + return failTtl ? { status: 500, body: "boom" } : { body: JSON.stringify({ engine_full: engine }) + "\n" }; + } + }); + const modifies = () => stub.calls.filter((c) => c.sql.includes("MODIFY TTL")); + try { + await new ClickHouseStore(config({ clickhouseUrl: stub.url, logTtlDays: 14 })).ensureSchema(); + assert.equal(modifies().length, 0); + await new ClickHouseStore(config({ clickhouseUrl: stub.url, logTtlDays: 30 })).ensureSchema(); + assert.equal(modifies().length, 1); + assert.match(modifies()[0]!.sql, /ALTER TABLE autter_runtime\.runtime_logs MODIFY TTL toDateTime\(occurred_at\) \+ INTERVAL 30 DAY/); + engine = engine.replace("toIntervalDay(14)", "toIntervalDay(30)"); + await new ClickHouseStore(config({ clickhouseUrl: stub.url, logTtlDays: 30 })).ensureSchema(); + assert.equal(modifies().length, 1); + failTtl = true; + await new ClickHouseStore(config({ clickhouseUrl: stub.url, logTtlDays: 7 })).ensureSchema(); + assert.equal(modifies().length, 1); + // Fresh databases: the baseline CREATE carries the configured TTL. + assert.ok(stub.calls.some((c) => /runtime_logs[\s\S]*INTERVAL 7 DAY/.test(c.sql))); + } finally { + await close(stub.server); + } +}); + +test("/v1/logs promotes logger-only errors with in-batch and ClickHouse dedupe", async () => { + const traceA = "a".repeat(32); + const traceB = "b".repeat(32); + let lookup: "ok" | "fail" = "ok"; + const inserted: Record>> = {}; + const ch = await startStub((call) => { + if (call.sql.includes("SELECT DISTINCT trace_id")) { + if (lookup === "fail") return { status: 500, body: "lookup down" }; + return { body: JSON.stringify({ trace_id: traceB }) + "\n" }; + } + const insert = /INSERT INTO autter_runtime\.(\w+)/.exec(call.params.get("query") ?? ""); + if (insert) { + (inserted[insert[1]!] ??= []).push( + ...call.body.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)), + ); + } + }); + const batches: Array> = []; + const sink = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + batches.push(JSON.parse(body)); + res.writeHead(202).end(); + }); + }); + await new Promise((resolve) => sink.listen(0, "127.0.0.1", resolve)); + const sinkUrl = `http://127.0.0.1:${(sink.address() as AddressInfo).port}/sink`; + const app = createIngesterApp(config({ clickhouseUrl: ch.url, sinkUrl })).app.listen(0, "127.0.0.1"); + await new Promise((resolve) => app.once("listening", resolve)); + const url = `http://127.0.0.1:${(app.address() as AddressInfo).port}`; + + const now = BigInt(Date.now()) * 1_000_000n; + const errorRecord = (traceId: string | undefined, message: string, extra: ReturnType[] = []) => ({ + timeUnixNano: String(now), + severityNumber: 17, + ...(traceId ? { traceId } : {}), + body: { stringValue: message }, + attributes: [ + str("autter.capture.mode", "log"), + str("exception.type", "RuntimeError"), + str("exception.message", message), + ...extra, + ], + }); + const post = (records: unknown[]) => + fetch(`${url}/v1/logs`, { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer server" }, + body: JSON.stringify({ + resourceLogs: [{ resource: { attributes: [str("service.name", "edge-api")] }, scopeLogs: [{ logRecords: records }] }], + }), + }); + const waitForBatches = async (n: number) => { + for (let i = 0; i < 100 && batches.length < n; i++) await new Promise((r) => setTimeout(r, 20)); + assert.equal(batches.length, n); + }; + + try { + const res = await post([ + errorRecord(traceA, "Payment declined", [ + str("autter.error.code", "billing.declined"), + str("autter.error.why", "Issuer rejected"), + str("autter.request.id", "req_abcdefgh12"), + str("http.route", "/pay"), + ]), + errorRecord(traceA, "Second error in the same trace"), // in-batch duplicate + errorRecord(traceB, "Already captured on the trace path"), // ClickHouse says seen + errorRecord(undefined, "No trace at all"), + { timeUnixNano: String(now), body: { stringValue: "plain info log" } }, + ]); + assert.equal(res.status, 200); + assert.equal(inserted.runtime_logs!.length, 5); + const occurrences = inserted.runtime_error_occurrences!; + assert.deepEqual(occurrences.map((o) => o.message), ["Payment declined", "No trace at all"]); + assert.equal(occurrences[0]!.error_code, "billing.declined"); + assert.equal(occurrences[0]!.fingerprint, codeFingerprint("edge-api", "billing.declined")); + assert.equal(occurrences[0]!.fingerprint_scheme, "code-v1"); + assert.equal(occurrences[0]!.error_why, "Issuer rejected"); + assert.equal(occurrences[0]!.request_id, "req_abcdefgh12"); + assert.equal(occurrences[0]!.source, "server"); + assert.equal(occurrences[1]!.fingerprint_scheme, "message-v1"); + const promotedLog = inserted.runtime_logs!.find((row) => row.message === "Payment declined")!; + assert.equal(promotedLog.error_code, "billing.declined"); + assert.equal(promotedLog.request_id, "req_abcdefgh12"); + // The lookup was scoped to the tenant and the candidate traces. + const query = ch.calls.find((c) => c.sql.includes("SELECT DISTINCT trace_id"))!; + assert.equal(query.params.get("param_org"), "org-1"); + assert.equal(query.params.get("param_repo"), "repo-1"); + assert.match(query.params.get("param_traces") ?? "", new RegExp(traceA)); + assert.ok(inserted.runtime_metrics_1m!.length > 0); + + await waitForBatches(1); + assert.deepEqual(batches[0]!.occurrences.map((o: any) => o.message), ["Payment declined", "No trace at all"]); + assert.equal(batches[0]!.occurrences[0].errorCode, "billing.declined"); + assert.equal(batches[0]!.occurrences[0].requestId, "req_abcdefgh12"); + assert.equal(batches[0]!.occurrences[0].traceId, traceA); + assert.equal(batches[0]!.occurrences[0].route, "/pay"); + assert.equal(batches[0]!.occurrences[0].fingerprintScheme, "code-v1"); + + // Lookup failure: promote anyway. + lookup = "fail"; + inserted.runtime_error_occurrences = []; + assert.equal((await post([errorRecord(traceB, "Lookup down but still promoted")])).status, 200); + assert.deepEqual(inserted.runtime_error_occurrences.map((o) => o.message), ["Lookup down but still promoted"]); + await waitForBatches(2); + } finally { + await close(app); + await close(ch.server); + await close(sink); + } +}); diff --git a/packages/otlp-ingester/src/config.ts b/packages/otlp-ingester/src/config.ts index 45d1d5c..a88d14a 100644 --- a/packages/otlp-ingester/src/config.ts +++ b/packages/otlp-ingester/src/config.ts @@ -38,6 +38,12 @@ export interface IngesterConfig { spanTtlDays: number; metricsTtlDays: number; llmCallTtlDays: number; + /** + * runtime_logs retention (LOG_TTL_DAYS, default 14). Request summaries are + * always kept (never sampled), so this is the main volume knob. Applied to + * fresh tables by the baseline and to existing ones at boot. + */ + logTtlDays: number; } function intEnv(name: string, fallback: number): number { @@ -91,6 +97,7 @@ export function loadConfig(): IngesterConfig { // LLM calls keep the metrics horizon, not the span one — cost trends // need months, and per-call volume is small next to HTTP spans. llmCallTtlDays: intEnv("LLM_CALL_TTL_DAYS", 90), + logTtlDays: intEnv("LOG_TTL_DAYS", 14), }; if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) { throw new Error( diff --git a/packages/otlp-ingester/src/context.ts b/packages/otlp-ingester/src/context.ts index ff5e30d..6659895 100644 --- a/packages/otlp-ingester/src/context.ts +++ b/packages/otlp-ingester/src/context.ts @@ -21,7 +21,9 @@ export function sanitizeRuntimeContext( if (--budget < 0 || depth > 6) return "[truncated]"; if (/__proto__|constructor|prototype/i.test(key)) return undefined; const usage = - /(?:^|\.)(?:input|output|total|prompt|completion)_?tokens$/i.test(key) && + // Numeric token COUNTS (incl. the AI rollup's cache_read_tokens) are + // usage, not secrets. + /(?:^|\.)(?:input|output|total|prompt|completion|cache_read|cache_creation)_?tokens$/i.test(key) && typeof value === "number" && Number.isFinite(value) && value >= 0; diff --git a/packages/otlp-ingester/src/error-fields.ts b/packages/otlp-ingester/src/error-fields.ts new file mode 100644 index 0000000..06f604a --- /dev/null +++ b/packages/otlp-ingester/src/error-fields.ts @@ -0,0 +1,107 @@ +import { sanitizeRuntimeContext } from "./context.js"; +import { validErrorCode } from "./fingerprint.js"; + +/** + * Declared error metadata on the wire (span/event attributes, log + * attributes, browser context keys): + * + * autter.error.code string matching CODE_PATTERN (≤80) — grouping key + * autter.error.why declared cause (≤1000 chars) + * autter.error.fix declared remedy (≤1000 chars) + * autter.error.link docs link (http(s) only, ≤500 chars) + * autter.error.expected boolean — expected business failure + * autter.request.id request id joining the error to its summary + * + * Every field is optional and validated independently: a bad value is + * dropped on its own and never rejects the signal. Values are lifted onto + * dedicated occurrence fields (and columns) instead of living only in the + * attributes JSON, because the generic context sanitiser rewrites URLs + * (query AND fragment stripped) — the declared link keeps its #anchor here. + */ + +export const REQUEST_ID_PATTERN = /^[\w.-]{8,128}$/; + +export interface DeclaredErrorFields { + errorCode?: string; + why?: string; + fix?: string; + link?: string; + expected?: boolean; + requestId?: string; +} + +type AttributeSource = + | Map + | Record + | null + | undefined; + +const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; + +function lookup(sources: AttributeSource[], key: string): unknown { + for (const source of sources) { + if (!source) continue; + const value = + source instanceof Map ? source.get(key) : (source as Record)[key]; + if (value !== undefined && value !== null && value !== "") return value; + } + return undefined; +} + +/** Free text through the same scrubber as custom context, then capped. */ +function declaredText(value: unknown, max: number): string | undefined { + if (typeof value !== "string") return undefined; + const text = sanitizeRuntimeContext({ text: value.trim() }).text; + return typeof text === "string" && text ? text.slice(0, max) : undefined; +} + +/** + * http(s) only; credentials and query string removed (a docs link never + * needs them), fragment kept. Over-long links are dropped rather than + * truncated — a cut URL is a broken link. + */ +export function declaredLink(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return undefined; + } + if (url.protocol !== "https:" && url.protocol !== "http:") return undefined; + url.username = ""; + url.password = ""; + url.search = ""; + const link = url.toString().replace(EMAIL_RE, "[redacted]"); + return link.length <= 500 ? link : undefined; +} + +export function validRequestId(value: unknown): string | undefined { + return typeof value === "string" && REQUEST_ID_PATTERN.test(value) + ? value + : undefined; +} + +/** + * Lift the declared fields from one or more attribute sources; the first + * source carrying a key wins (e.g. exception event before its span). + * Returns only the fields that are present and valid. + */ +export function liftErrorFields( + ...sources: AttributeSource[] +): DeclaredErrorFields { + const out: DeclaredErrorFields = {}; + const code = validErrorCode(lookup(sources, "autter.error.code")); + if (code) out.errorCode = code; + const why = declaredText(lookup(sources, "autter.error.why"), 1000); + if (why) out.why = why; + const fix = declaredText(lookup(sources, "autter.error.fix"), 1000); + if (fix) out.fix = fix; + const link = declaredLink(lookup(sources, "autter.error.link")); + if (link) out.link = link; + const expected = lookup(sources, "autter.error.expected"); + if (expected === true || expected === "true") out.expected = true; + const requestId = validRequestId(lookup(sources, "autter.request.id")); + if (requestId) out.requestId = requestId; + return out; +} diff --git a/packages/otlp-ingester/src/fingerprint.ts b/packages/otlp-ingester/src/fingerprint.ts index 11329ee..d844758 100644 --- a/packages/otlp-ingester/src/fingerprint.ts +++ b/packages/otlp-ingester/src/fingerprint.ts @@ -271,7 +271,62 @@ export function normalizeStackFrames( return frames.length > 0 ? frames : fallbackFrames(lines, topN); } +/** + * Error codes (`autter.error.code`) — namespaced, stable, low-cardinality + * identifiers such as "billing.declined". Anything else (Node's "ECONNRESET", + * ids, free text) is not a code and the error groups by message as before. + * Shared spec: runtime-core and the Autter backend validate identically. + */ +export const CODE_PATTERN = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$/; +export const CODE_MAX_LENGTH = 80; + +export function validErrorCode(value: unknown): string | undefined { + return typeof value === "string" && + value.length <= CODE_MAX_LENGTH && + CODE_PATTERN.test(value) + ? value + : undefined; +} + +export type FingerprintScheme = "message-v1" | "code-v1"; + +/** + * Scheme "code-v1": a SOURCE-INDEPENDENT issue identity for coded errors. + * Source, error type, message, stack and route are deliberately left out so + * the same code from the trace path, log promotion, the browser or an + * external connector (Sentry, PostHog, …) lands in ONE issue. Route and + * message stay as facets on the occurrence rows. Mirrored byte-for-byte by + * the backend (`runtime-code-fingerprint.ts`) — shared test vectors pin it. + */ +export function codeFingerprint(service: string, code: string): string { + return createHash("sha256") + .update(`code-v1\u0000${service}\u0000${code}`) + .digest("hex") + .slice(0, 32); +} + +/** + * Issue identity plus the scheme that produced it. A valid + * `autter.error.code` selects "code-v1"; everything else keeps the + * historical message/stack/route fingerprint ("message-v1") byte-for-byte, + * so existing issues never regroup. + */ +export function occurrenceFingerprint(input: RuntimeOccurrenceInput): { + fingerprint: string; + scheme: FingerprintScheme; +} { + const code = validErrorCode(input.errorCode); + return code + ? { fingerprint: codeFingerprint(input.service, code), scheme: "code-v1" } + : { fingerprint: messageFingerprint(input), scheme: "message-v1" }; +} + export function fingerprintOccurrence(input: RuntimeOccurrenceInput): string { + return occurrenceFingerprint(input).fingerprint; +} + +/** Scheme "message-v1" — the pre-1.5.0 fingerprint, unchanged. */ +function messageFingerprint(input: RuntimeOccurrenceInput): string { const parts = [ input.source, input.service, diff --git a/packages/otlp-ingester/src/log-promotion.ts b/packages/otlp-ingester/src/log-promotion.ts new file mode 100644 index 0000000..4554401 --- /dev/null +++ b/packages/otlp-ingester/src/log-promotion.ts @@ -0,0 +1,100 @@ +import { normalizeRoute } from "./fingerprint.js"; +import type { RuntimeLogRecord } from "./logs.js"; +import type { + RuntimeMetricPoint, + RuntimeOccurrence, + RuntimeOccurrenceInput, +} from "./types.js"; + +/** + * Log promotion — errors from SDKs running WITHOUT a trace pipeline. + * + * Logger-only mode (`initAutterLogging`) and `@autter/runtime-edge` have no + * span to record an exception on, so they emit an error log record with + * `exception.*` and `autter.capture.mode = "log"`. /v1/logs turns those + * records into ordinary server occurrences (fingerprinted, stored in + * runtime_error_occurrences, forwarded to the sink) — otherwise their + * errors would never reach issue grouping. + * + * Dedupe, so one failure never becomes two occurrences through two export + * paths: + * - in-batch: at most one promoted occurrence per trace id (first wins); + * - cross-batch: a trace id that already produced an occurrence for this + * tenant within ±PROMOTION_DEDUPE_WINDOW_MS (ClickHouse lookup) is skipped. + * Records without a trace id are always promoted. The lookup is + * best-effort: when it fails the record is promoted anyway — a rare + * duplicate beats a silently dropped error. + */ + +export const PROMOTION_DEDUPE_WINDOW_MS = 60_000; + +/** Promotion candidates in record order, deduped by trace id in-batch. */ +export function logPromotionCandidates( + rows: RuntimeLogRecord[], +): RuntimeOccurrenceInput[] { + const traces = new Set(); + const out: RuntimeOccurrenceInput[] = []; + for (const row of rows) { + const occurrence = row.occurrence; + if (!occurrence) continue; + if (occurrence.traceId) { + if (traces.has(occurrence.traceId)) continue; + traces.add(occurrence.traceId); + } + out.push(occurrence); + } + return out; +} + +/** Lookup window around the candidates: [earliest − 60 s, latest + 60 s]. */ +export function promotionLookupWindow( + occurrences: RuntimeOccurrenceInput[], +): { from: Date; to: Date } { + const times = occurrences.map((o) => o.occurredAt.getTime()); + return { + from: new Date(Math.min(...times) - PROMOTION_DEDUPE_WINDOW_MS), + to: new Date(Math.max(...times) + PROMOTION_DEDUPE_WINDOW_MS), + }; +} + +/** + * One event / one error event per promoted occurrence — the same counting + * the trace path applies to occurrences outside a server span, so issue + * counters for logger-only services don't read 0. + */ +export function promotionRollups( + occurrences: RuntimeOccurrence[], +): RuntimeMetricPoint[] { + const rollups = new Map(); + for (const occ of occurrences) { + const bucketAt = new Date( + Math.floor(occ.occurredAt.getTime() / 60_000) * 60_000, + ); + const route = normalizeRoute(occ.route); + const key = [ + occ.service, + occ.environment, + occ.release ?? "", + route, + bucketAt.getTime(), + ].join(" "); + const existing = rollups.get(key); + if (existing) { + existing.requestCount += 1; + existing.errorCount += 1; + continue; + } + rollups.set(key, { + service: occ.service, + environment: occ.environment, + release: occ.release, + route, + bucketAt, + requestCount: 1, + errorCount: 1, + durationSumMs: 0, + sessionCount: 0, + }); + } + return [...rollups.values()]; +} diff --git a/packages/otlp-ingester/src/logs.ts b/packages/otlp-ingester/src/logs.ts index ec0ffaf..e67277b 100644 --- a/packages/otlp-ingester/src/logs.ts +++ b/packages/otlp-ingester/src/logs.ts @@ -5,7 +5,9 @@ import { type OtlpAttribute, type OtlpValue, } from "./context.js"; +import { declaredLink, liftErrorFields } from "./error-fields.js"; import { normalizeRoute } from "./fingerprint.js"; +import type { RuntimeOccurrenceInput } from "./types.js"; export interface RuntimeLogRecord { id: string; @@ -23,6 +25,28 @@ export interface RuntimeLogRecord { durationMs: number; attributes: Record; occurredAt: Date; + /** + * Lifted wide-event columns (migration 0012). `kind` is "request" for + * request summaries, "operation" for operation summaries (including + * pre-1.5.0 SDKs that never sent `autter.operation.kind`), "" for plain + * log records. The rest default to ""/0 when the record doesn't carry them. + */ + kind: "" | "request" | "operation"; + requestId: string; + /** `http.route`, query-stripped and id-normalised. */ + route: string; + statusCode: number; + /** `autter.error.code`, only when it matches CODE_PATTERN. */ + errorCode: string; + aiCostUsd: number; + aiCalls: number; + /** + * Not persisted on runtime_logs. Set for logger-only/edge error records + * (`autter.capture.mode = "log"`, severity ≥ error, `exception.*` + * present): the occurrence /v1/logs promotes into issue grouping — + * see log-promotion.ts. + */ + occurrence?: RuntimeOccurrenceInput; } export interface OtlpLogsRequest { resourceLogs?: Array<{ @@ -132,6 +156,66 @@ export function normalizeLogs(request: OtlpLogsRequest): RuntimeLogRecord[] { : ""; if (typeof attributes.path === "string") attributes.path = normalizeRoute(attributes.path.split(/[?#]/)[0]!); + const service = String(info["service.name"] ?? "unknown").slice(0, 200); + const environment = String( + info["deployment.environment.name"] ?? + info["deployment.environment"] ?? + "production", + ).slice(0, 100); + const release = String(info["service.version"] ?? "").slice(0, 200); + const declared = liftErrorFields(attributes); + // The sanitiser strips URL fragments; the declared docs link keeps + // its #anchor from the raw attribute (still validated). + const rawLink = declaredLink( + record.attributes?.find((a) => a?.key === "autter.error.link") + ?.value?.stringValue, + ); + if (rawLink) declared.link = rawLink; + const rawRoute = + typeof attributes["http.route"] === "string" + ? attributes["http.route"].split(/[?#]/)[0]! + : ""; + const route = normalizeRoute(rawRoute).slice(0, 500); + const status = Number(attributes["http.response.status_code"]); + const statusCode = + Number.isInteger(status) && status > 0 && status < 1000 ? status : 0; + const declaredKind = attributes["autter.operation.kind"]; + const type = + attributes["autter.event.type"] === "operation" ? "operation" : "log"; + const ai = attributes["autter.operation.ai"]; + const aiRollup = + ai && typeof ai === "object" && !Array.isArray(ai) + ? (ai as Record) + : {}; + const occurrence = + attributes["autter.capture.mode"] === "log" && + (severity === "error" || severity === "fatal") && + hasException(attributes) + ? { + source: "server" as const, + severity: severity as "error" | "fatal", + service, + environment, + release: release || null, + errorType: String(attributes["exception.type"] || "Error").slice(0, 200), + message: String(attributes["exception.message"] || message || "Unknown error").slice(0, 4000), + stack: + typeof attributes["exception.stacktrace"] === "string" + ? attributes["exception.stacktrace"] + : null, + route: rawRoute || stringOrNull(attributes["url.path"]), + method: + typeof attributes["http.request.method"] === "string" + ? attributes["http.request.method"].toUpperCase().slice(0, 16) + : null, + statusCode: statusCode || null, + traceId: traceId || null, + sessionId: null, + attributes, + occurredAt, + ...declared, + } + : undefined; rows.push({ id: createHash("sha256") .update( @@ -147,13 +231,9 @@ export function normalizeLogs(request: OtlpLogsRequest): RuntimeLogRecord[] { ) .digest("hex") .slice(0, 32), - service: String(info["service.name"] ?? "unknown").slice(0, 200), - environment: String( - info["deployment.environment.name"] ?? - info["deployment.environment"] ?? - "production", - ).slice(0, 100), - release: String(info["service.version"] ?? "").slice(0, 200), + service, + environment, + release, traceId, spanId, operationId, @@ -161,10 +241,7 @@ export function normalizeLogs(request: OtlpLogsRequest): RuntimeLogRecord[] { 0, 200, ), - type: - attributes["autter.event.type"] === "operation" - ? "operation" - : "log", + type, severity, message, outcome: String(attributes["autter.operation.outcome"] ?? "").slice( @@ -177,6 +254,26 @@ export function normalizeLogs(request: OtlpLogsRequest): RuntimeLogRecord[] { ), attributes, occurredAt, + kind: + declaredKind === "request" || declaredKind === "operation" + ? declaredKind + : type === "operation" + ? "operation" + : "", + requestId: declared.requestId ?? "", + route, + statusCode, + errorCode: declared.errorCode ?? "", + aiCostUsd: nonNegative( + attributes["autter.operation.ai.cost_usd"] ?? aiRollup.cost_usd, + ), + aiCalls: Math.min( + 4_294_967_295, + Math.round( + nonNegative(attributes["autter.operation.ai.calls"] ?? aiRollup.calls), + ), + ), + ...(occurrence ? { occurrence } : {}), }); } } @@ -184,7 +281,28 @@ export function normalizeLogs(request: OtlpLogsRequest): RuntimeLogRecord[] { return rows; } -export function logTableDDL(db: string): string { +function hasException(attributes: Record): boolean { + return ["exception.type", "exception.message", "exception.stacktrace"].some( + (key) => typeof attributes[key] === "string" && attributes[key] !== "", + ); +} + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value ? value.split(/[?#]/)[0]! : null; +} + +function nonNegative(value: unknown): number { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : 0; +} + +/** + * Baseline for fresh databases. The 0012 columns and request-id skip index + * are included so a fresh DB matches a migrated one. `ttlDays` is + * LOG_TTL_DAYS; existing tables are brought in line at boot + * (ClickHouseStore.applyLogTtl), not here. + */ +export function logTableDDL(db: string, ttlDays = 14): string { return `CREATE TABLE IF NOT EXISTS ${db}.runtime_logs ( org_id String, repository_id String, event_id String, service LowCardinality(String), environment LowCardinality(String), release String DEFAULT '', @@ -193,8 +311,72 @@ export function logTableDDL(db: string): string { severity LowCardinality(String) DEFAULT 'info', message String CODEC(ZSTD(1)), outcome LowCardinality(String) DEFAULT '', duration_ms Float64 DEFAULT 0, attributes String DEFAULT '{}' CODEC(ZSTD(1)), occurred_at DateTime64(3, 'UTC'), - ingested_at DateTime64(3, 'UTC') DEFAULT now64(3) + ingested_at DateTime64(3, 'UTC') DEFAULT now64(3), + ${LOG_REQUEST_COLUMNS.map((column) => column.ddl).join(", ")}, + ${LOG_REQUEST_ID_INDEX} ) ENGINE = ReplacingMergeTree PARTITION BY toDate(occurred_at) ORDER BY (org_id, repository_id, occurred_at, event_id) - TTL toDateTime(occurred_at) + INTERVAL 14 DAY`; + TTL toDateTime(occurred_at) + INTERVAL ${ttlDays} DAY`; +} + +/** Migration 0012 columns — shared by the baseline DDL and the migration. */ +export const LOG_REQUEST_COLUMNS: Array<{ name: string; ddl: string }> = [ + { name: "kind", ddl: "kind LowCardinality(String) DEFAULT ''" }, + { name: "request_id", ddl: "request_id String DEFAULT ''" }, + { name: "route", ddl: "route String DEFAULT ''" }, + { name: "status_code", ddl: "status_code UInt16 DEFAULT 0" }, + { name: "error_code", ddl: "error_code String DEFAULT ''" }, + { name: "ai_cost_usd", ddl: "ai_cost_usd Float64 DEFAULT 0" }, + { name: "ai_calls", ddl: "ai_calls UInt32 DEFAULT 0" }, +]; +export const LOG_REQUEST_ID_INDEX = + "INDEX idx_logs_request_id request_id TYPE bloom_filter GRANULARITY 4"; + +/** + * Per-minute request rollup (migration 0014), fed by runtime_request_1m_mv + * from `runtime_logs` rows with kind = 'request'. Route stats read this + * instead of scanning raw summaries: + * sum(request_count), sum(failed_count), sum(duration_sum_ms), + * quantilesMerge(0.5, 0.95)(duration_quantiles) + */ +export function requestRollupTableDDL(db: string): string { + return `CREATE TABLE IF NOT EXISTS ${db}.runtime_request_1m ( + org_id String, repository_id String, + service LowCardinality(String), environment LowCardinality(String), + route String, method LowCardinality(String), bucket_at DateTime('UTC'), + request_count SimpleAggregateFunction(sum, UInt64), + failed_count SimpleAggregateFunction(sum, UInt64), + duration_sum_ms SimpleAggregateFunction(sum, Float64), + duration_quantiles AggregateFunction(quantiles(0.5, 0.95), Float64) + ) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(bucket_at) + ORDER BY (org_id, repository_id, service, environment, route, method, bucket_at) + TTL bucket_at + INTERVAL 90 DAY`; +} + +/** + * The view only sees rows inserted AFTER it exists (no backfill) and is + * created by migration 0014 only — not the baseline — because it reads the + * 0012 columns, which an existing runtime_logs table lacks until migrations + * run. runtime_logs is a ReplacingMergeTree keyed by a content-derived + * event_id, so a retried batch collapses there on merge — but the view + * aggregates every INSERT, so retries can count a summary twice. The + * rollup is therefore approximate under exporter retries (rare: a retry + * needs a 503 or a lost 2xx); exact per-request answers come from + * runtime_logs itself. `method` is read from the attributes JSON since + * runtime_logs has no method column. + */ +export function requestRollupViewDDL(db: string): string { + return `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.runtime_request_1m_mv + TO ${db}.runtime_request_1m AS + SELECT + org_id, repository_id, service, environment, route, + upper(JSONExtractString(attributes, 'http.request.method')) AS method, + toDateTime(toStartOfMinute(occurred_at), 'UTC') AS bucket_at, + count() AS request_count, + countIf(outcome = 'failed') AS failed_count, + sum(duration_ms) AS duration_sum_ms, + quantilesState(0.5, 0.95)(duration_ms) AS duration_quantiles + FROM ${db}.runtime_logs + WHERE kind = 'request' + GROUP BY org_id, repository_id, service, environment, route, method, bucket_at`; } diff --git a/packages/otlp-ingester/src/migrations.ts b/packages/otlp-ingester/src/migrations.ts index d1f0a9f..05474d6 100644 --- a/packages/otlp-ingester/src/migrations.ts +++ b/packages/otlp-ingester/src/migrations.ts @@ -36,9 +36,28 @@ import { latencyTableDDL } from "./latency.js"; import { profileTableDDL } from "./profiles.js"; import { sourceMapTableDDL } from "./source-maps.js"; -import { logTableDDL } from "./logs.js"; +import { + LOG_REQUEST_COLUMNS, + LOG_REQUEST_ID_INDEX, + logTableDDL, + requestRollupTableDDL, + requestRollupViewDDL, +} from "./logs.js"; import { memoryTableDDL, platformEventTableDDL } from "./memory.js"; +/** Migration 0013 columns — shared with the baseline DDL in clickhouse.ts. */ +export const OCCURRENCE_CODE_COLUMNS: string[] = [ + "error_code String DEFAULT ''", + "error_why String DEFAULT ''", + "error_fix String DEFAULT ''", + "error_link String DEFAULT ''", + "expected UInt8 DEFAULT 0", + "request_id String DEFAULT ''", + "fingerprint_scheme LowCardinality(String) DEFAULT 'message-v1'", +]; +export const OCCURRENCE_REQUEST_ID_INDEX = + "INDEX idx_occ_request_id request_id TYPE bloom_filter GRANULARITY 4"; + export interface Migration { /** Unique, ordered id: "-". Never reuse or reorder. */ id: string; @@ -137,6 +156,39 @@ export const MIGRATIONS: Migration[] = [ `ALTER TABLE {db}.runtime_memory_samples ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'gauge'`, ] }, { id: "0011-runtime-logs", statements: [logTableDDL("{db}")] }, + // Request wide events: request summaries share runtime_logs with + // operations and plain logs; the lifted columns make "requests on this + // route", "everything for request X" and "coded failures" plain column + // filters. The skip index only covers parts written after it exists + // (no MATERIALIZE INDEX mutation) — older rows predate request ids. + { + id: "0012-runtime-logs-requests", + statements: [ + ...LOG_REQUEST_COLUMNS.map( + (column) => + `ALTER TABLE {db}.runtime_logs ADD COLUMN IF NOT EXISTS ${column.ddl}`, + ), + `ALTER TABLE {db}.runtime_logs ADD ${LOG_REQUEST_ID_INDEX.replace("INDEX ", "INDEX IF NOT EXISTS ")}`, + ], + }, + // Declared error metadata and code-based grouping (fingerprint_scheme + // "code-v1" vs the historical "message-v1"). + { + id: "0013-occurrence-codes", + statements: [ + ...OCCURRENCE_CODE_COLUMNS.map( + (column) => + `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`, + ), + `ALTER TABLE {db}.runtime_error_occurrences ADD ${OCCURRENCE_REQUEST_ID_INDEX.replace("INDEX ", "INDEX IF NOT EXISTS ")}`, + ], + }, + // Per-minute route rollup of request summaries. Table first, then the + // view (which needs both the table and the 0012 columns). + { + id: "0014-runtime-request-1m", + statements: [requestRollupTableDDL("{db}"), requestRollupViewDDL("{db}")], + }, ]; /** The tracking table itself — created by the runner before anything else. */ diff --git a/packages/otlp-ingester/src/normalize-browser.ts b/packages/otlp-ingester/src/normalize-browser.ts index 847d58e..c282049 100644 --- a/packages/otlp-ingester/src/normalize-browser.ts +++ b/packages/otlp-ingester/src/normalize-browser.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { liftErrorFields } from "./error-fields.js"; import { normalizeRoute } from "./fingerprint.js"; import { asSeverity, @@ -41,6 +42,14 @@ const browserEventSchema = z.object({ column: z.number().int().nonnegative().optional(), /** Path only; query strings are stripped defensively anyway. */ route: z.string().max(1000).optional(), + /** + * Free-form, scrubbed context. runtime-browser ≥1.4.0 also sends the + * declared-error keys `autter.error.code|why|fix|link|expected` (from + * coded errors / `autterErrorFromResponse`) and `autter.request.id` (the + * `x-request-id` of a failed fetch/XHR). They are validated one by one + * in normalizeBrowserPayload — a malformed value is dropped on its own + * instead of rejecting the whole beacon. + */ context: z.record(z.unknown()).optional(), }); @@ -198,6 +207,7 @@ export function normalizeBrowserPayload( ...(event.context ? { context: scrubContext(event.context) } : {}), }, occurredAt, + ...liftErrorFields(event.context), }); } diff --git a/packages/otlp-ingester/src/normalize-otlp.ts b/packages/otlp-ingester/src/normalize-otlp.ts index e939916..4be8550 100644 --- a/packages/otlp-ingester/src/normalize-otlp.ts +++ b/packages/otlp-ingester/src/normalize-otlp.ts @@ -1,4 +1,5 @@ import { decodeOtlpAttributes, sanitizeRuntimeContext, type OtlpValue } from "./context.js"; +import { liftErrorFields, validRequestId } from "./error-fields.js"; import { normalizeRoute } from "./fingerprint.js"; import { extractLlmCall } from "./llm.js"; import { @@ -294,6 +295,9 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes), ...(handled ? { "autter.handled": true, "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) }), occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano), + // Event attributes first: one span can carry several + // exceptions, each with its own code. + ...liftErrorFields(eventAttrs, attrs), }); } const failedOutcomes = (span.events ?? []).filter((event) => { @@ -313,6 +317,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { traceId: span.traceId ?? null, sessionId: null, attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }), occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano), + ...liftErrorFields(outcome, attrs), }); } if (exceptionEvents.length === 0 && failedOutcomes.length === 0 && isError) { @@ -335,6 +340,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { sessionId: null, attributes: decodeOtlpAttributes(span.attributes), occurredAt: startedAt, + ...liftErrorFields(attrs), }); } @@ -435,6 +441,10 @@ function inheritHttpFromParent( const http = httpServerAncestor(byId, spanId); if (!http) continue; if (!occurrence.route && http.route) occurrence.route = http.route; + // Request ids live on the request (server) span; child error spans + // from captureException inherit it so the issue links to the request. + const requestId = validRequestId(http.attributes?.["autter.request.id"]); + if (!occurrence.requestId && requestId) occurrence.requestId = requestId; const method = methodFromSpan(http); if (!occurrence.method && method) occurrence.method = method; if ( diff --git a/packages/otlp-ingester/src/server.logs.test.ts b/packages/otlp-ingester/src/server.logs.test.ts index d6fe642..54ac3a9 100644 --- a/packages/otlp-ingester/src/server.logs.test.ts +++ b/packages/otlp-ingester/src/server.logs.test.ts @@ -69,6 +69,7 @@ test("log ingestion authenticates, preserves tenant mapping and refuses malforme spanTtlDays: 7, metricsTtlDays: 90, llmCallTtlDays: 90, + logTtlDays: 14, }; const app = createIngesterApp(config).app.listen(); const url = await new Promise((resolve) => diff --git a/packages/otlp-ingester/src/server.sink.test.ts b/packages/otlp-ingester/src/server.sink.test.ts index af22119..7166ae5 100644 --- a/packages/otlp-ingester/src/server.sink.test.ts +++ b/packages/otlp-ingester/src/server.sink.test.ts @@ -82,6 +82,7 @@ before(async () => { spanTtlDays: 7, metricsTtlDays: 90, llmCallTtlDays: 90, + logTtlDays: 14, }; const { app } = createIngesterApp(config); appServer = app.listen(0); diff --git a/packages/otlp-ingester/src/server.ts b/packages/otlp-ingester/src/server.ts index 24d94c4..2dbda59 100644 --- a/packages/otlp-ingester/src/server.ts +++ b/packages/otlp-ingester/src/server.ts @@ -9,9 +9,14 @@ import { normalizeLatencyHistograms } from "./latency.js"; import type { IngesterConfig } from "./config.js"; import { deriveFields, - fingerprintOccurrence, + occurrenceFingerprint, occurrenceIdFor, } from "./fingerprint.js"; +import { + logPromotionCandidates, + promotionLookupWindow, + promotionRollups, +} from "./log-promotion.js"; import { browserPayloadSchema, normalizeBrowserPayload, @@ -176,16 +181,51 @@ export function createIngesterApp(config: IngesterConfig): IngesterApp { inputs: RuntimeOccurrenceInput[], ): RuntimeOccurrence[] { return inputs.map((input, index) => { - const fingerprint = fingerprintOccurrence(input); + const { fingerprint, scheme } = occurrenceFingerprint(input); return { ...input, occurrenceId: occurrenceIdFor(ctx, input, fingerprint, index), fingerprint, + fingerprintScheme: scheme, ...deriveFields(input), }; }); } + /** Fingerprint log-promotion candidates and drop the ones whose trace + * already produced an occurrence (in this batch, or within ±60 s). Ids + * are assigned BEFORE the lookup so they stay stable across retries + * whatever the lookup returns. */ + async function promoteLogs( + ctx: IngestContext, + logs: ReturnType, + ): Promise { + const candidates = logPromotionCandidates(logs); + if (candidates.length === 0) return []; + const fingerprinted = fingerprintAll(ctx, candidates); + const traceIds = [ + ...new Set(fingerprinted.map((o) => o.traceId).filter((t): t is string => !!t)), + ]; + if (traceIds.length === 0) return fingerprinted; + let seen = new Set(); + try { + const { from, to } = promotionLookupWindow(candidates); + seen = await store.recentOccurrenceTraceIds( + ctx, + traceIds, + from, + to, + fingerprinted.map((o) => o.occurrenceId), + ); + } catch (err) { + console.warn( + "log promotion dedupe lookup failed — promoting without it:", + err instanceof Error ? err.message : err, + ); + } + return fingerprinted.filter((o) => !o.traceId || !seen.has(o.traceId)); + } + function storageError(res: Response, err: unknown): void { console.error("clickhouse write failed:", err); res.status(503).json({ error: "storage unavailable, retry later" }); @@ -298,10 +338,21 @@ export function createIngesterApp(config: IngesterConfig): IngesterApp { let logs; try { logs = normalizeLogs(req.is("application/x-protobuf") ? decodeLogsRequest(req.body as Buffer) : req.body as OtlpLogsRequest); } catch { res.status(400).json({ error: "invalid OTLP logs payload" }); return; } - try { await store.insertLogs(ctx, logs); } + // Logs are diagnostic evidence; exceptions and failed outcomes normally + // reach grouping through the trace path. The exception: logger-only / + // edge SDKs mark span-less error records `autter.capture.mode = "log"`, + // and those are promoted to occurrences here — see log-promotion.ts. + const promoted = await promoteLogs(ctx, logs); + const metricPoints = promotionRollups(promoted); + try { + await Promise.all([ + store.insertLogs(ctx, logs), + store.insertOccurrences(ctx, promoted), + store.insertMetricPoints(ctx, metricPoints), + ]); + } catch (err) { storageError(res, err); return; } - // Logs are diagnostic evidence. Exceptions and failed outcomes use the trace sink, - // so one operation does not create duplicate issues through two export paths. + sink?.enqueue(ctx, promoted, metricPoints); otlpSuccess(req, res); }); diff --git a/packages/otlp-ingester/src/sink.test.ts b/packages/otlp-ingester/src/sink.test.ts index 5367de7..db61447 100644 --- a/packages/otlp-ingester/src/sink.test.ts +++ b/packages/otlp-ingester/src/sink.test.ts @@ -39,6 +39,7 @@ function cfg(overrides: Partial = {}): IngesterConfig { spanTtlDays: 7, metricsTtlDays: 90, llmCallTtlDays: 90, + logTtlDays: 14, ...overrides, }; } @@ -66,6 +67,7 @@ function occ(overrides: Partial = {}): RuntimeOccurrence { occurredAt: new Date("2026-01-01T00:00:00.000Z"), occurrenceId: "occ-1", fingerprint: "fp-1", + fingerprintScheme: "message-v1", routeNormalized: "/x", messageNormalized: "boom", topFrames: [], diff --git a/packages/otlp-ingester/src/sink.ts b/packages/otlp-ingester/src/sink.ts index eac21c6..8de6ec4 100644 --- a/packages/otlp-ingester/src/sink.ts +++ b/packages/otlp-ingester/src/sink.ts @@ -203,10 +203,7 @@ export class SinkForwarder { batchId, orgId: ctx.orgId, repositoryId: ctx.repositoryId, - occurrences: occurrences.map((o) => ({ - ...o, - occurredAt: o.occurredAt.toISOString(), - })), + occurrences: occurrences.map(sinkOccurrence), metrics: metricPoints.map((p) => ({ ...p, bucketAt: p.bucketAt.toISOString(), @@ -412,6 +409,44 @@ export class SinkForwarder { } } +/** + * Wire shape of one occurrence (payload `version: 1`). 1.5.0 adds, all + * optional and omitted when absent: errorCode, why, fix, link, expected, + * requestId, traceId, route, method, statusCode, fingerprintScheme. The + * request-context four were previously spread as `null` when unknown and + * are now simply left out, matching the documented optional-string shape. + * Older consumers ignore unknown keys, so the version stays 1. + */ +export function sinkOccurrence(o: RuntimeOccurrence): Record { + const { + traceId, + route, + method, + statusCode, + errorCode, + why, + fix, + link, + expected, + requestId, + ...rest + } = o; + return { + ...rest, + occurredAt: o.occurredAt.toISOString(), + ...(traceId ? { traceId } : {}), + ...(route ? { route } : {}), + ...(method ? { method } : {}), + ...(statusCode ? { statusCode } : {}), + ...(errorCode ? { errorCode } : {}), + ...(why ? { why } : {}), + ...(fix ? { fix } : {}), + ...(link ? { link } : {}), + ...(expected ? { expected: true } : {}), + ...(requestId ? { requestId } : {}), + }; +} + /** ISO range across every signal in the batch — the replay hint on drops. */ function signalRange( occurrences: RuntimeOccurrence[], diff --git a/packages/otlp-ingester/src/types.ts b/packages/otlp-ingester/src/types.ts index feced4c..0db32b5 100644 --- a/packages/otlp-ingester/src/types.ts +++ b/packages/otlp-ingester/src/types.ts @@ -44,6 +44,22 @@ export interface RuntimeOccurrenceInput { sessionId: string | null; attributes: Record | null; occurredAt: Date; + /** + * Declared error metadata (`autter.error.*`, see error-fields.ts) — + * optional and omitted when absent. A valid `errorCode` switches the + * fingerprint to the source-independent "code-v1" scheme. + */ + errorCode?: string; + /** Declared cause (≤1000 chars). */ + why?: string; + /** Declared remedy (≤1000 chars). */ + fix?: string; + /** Docs link — http(s) only, ≤500 chars. */ + link?: string; + /** Expected business failure (declines, validation): recorded, never paged. */ + expected?: boolean; + /** `autter.request.id` — joins the occurrence to its request summary. */ + requestId?: string; } /** @@ -54,6 +70,8 @@ export interface RuntimeOccurrenceInput { export interface RuntimeOccurrence extends RuntimeOccurrenceInput { occurrenceId: string; fingerprint: string; + /** "code-v1" when grouped by a valid error code, else "message-v1". */ + fingerprintScheme: "message-v1" | "code-v1"; /** normalizeRoute(route) — "/users/:id/orders/:id"; GROUP BY-safe. */ routeNormalized: string; /** normalizeMessage(message) — ids/numbers/strings templated out. */ diff --git a/packages/runtime-browser/README.md b/packages/runtime-browser/README.md index de33b5f..33186a8 100644 --- a/packages/runtime-browser/README.md +++ b/packages/runtime-browser/README.md @@ -17,8 +17,10 @@ can use release keyed source maps uploaded by CI; see `docs/CONTINUOUS-DETECTION.md` in the repository. Tiny, dependency-free browser error + usage tracker for Autter Runtime. -**~1 KB brotlied** (5 KB CI budget), zero runtime dependencies, no OTel SDK, -no console patching, no DOM recording, no offline storage. +**~4 KB brotlied** (5 KB CI budget), zero runtime dependencies, no OTel SDK, +no console patching, no DOM recording, no offline storage. ESM only (the +package has no CommonJS build; bundlers and `require(esm)`-capable Node +resolve the `default` export condition). ## Install @@ -71,12 +73,51 @@ initAutterBrowser({ }); ``` +## Coded errors and request ids (1.4.0) + +`captureException` — and the automatic error and rejection listeners — read +declared fields off **any** thrown value, so existing error classes with a +`code` property benefit without changes: + +| Error property | Sent as | Rule | +| --- | --- | --- | +| `code` | `autter.error.code` | Must match `^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$`, ≤80 chars (`billing.declined`); otherwise not sent (`ECONNRESET` is not a code) | +| `why` / `fix` | `autter.error.why` / `.fix` | Declared cause / remedy, first 1000 chars | +| `link` | `autter.error.link` | `http(s)://` only, ≤500 chars | +| `expected` | `autter.error.expected` | `true` marks an expected business failure (recorded, never pages) | +| `requestId` | `autter.request.id` | `^[\w.-]{8,128}$` | + +A valid code groups the error by **code** (one code = one issue across the +browser, the server SDKs and external sources) instead of by message. + +`autterErrorFromResponse(response)` turns a failed `fetch` Response into such +an error. It reads the `{ "error": { message, code, why, fix, link, requestId } }` +body produced by `autterErrorResponse()` / `toClientError()` in +`@autter/runtime-node` (from a clone — you can still read the body), and falls +back to the status text and the `x-request-id` header: + +```ts +const res = await fetch("/api/checkout", { method: "POST", body }); +if (!res.ok) { + const error = await autterErrorFromResponse(res); // name "HttpResponseError", .status, .code, … + captureException(error); + throw error; +} +``` + +Failed requests observed automatically (fetch/XHR 5xx) carry the response's +`x-request-id` header as `autter.request.id`, so the browser failure links to +the server's request summary without browser tracing. Cross-origin APIs must +expose the header (`Access-Control-Expose-Headers: x-request-id`) for the +page to read it. + ## API | Function | Notes | | --- | --- | | `initAutterBrowser(options)` | Installs error, rejection, CSP, and recent-action listeners; sends a session ping | -| `captureException(error, context?)` | Handled errors; fast-flushed | +| `captureException(error, context?)` | Handled errors; fast-flushed; duck-types `code/why/fix/link/expected/requestId` | +| `autterErrorFromResponse(response)` | `Promise` from a failed Response's JSON error body (or status text) | | `captureMessage(message, severity?, context?)` | Warnings/info without an exception (`"warning"` default); grouped and aggregated like errors | | `trackEvent(name, props?)` | Usage counter; aggregated server-side per minute | | `setUser(id)` | **Opaque id only** — never an email | @@ -99,7 +140,8 @@ localStorage, DOM text, form values, request headers/bodies, console history, IP addresses, and CSP policy text. Routes are `location.pathname` only. Browser and OS are a family plus major version. Filenames and script URLs are query-stripped. For CSP blocks, the directive, blocked resource origin, script -path, and a short policy hash are retained. +path, and a short policy hash are retained. Of response headers, only +`x-request-id` is read, and only on failed (5xx) requests. Custom `context` is free-form, so it is scrubbed before send: values under sensitive-looking keys (`email`, `password`, `token`, `secret`, `auth`, diff --git a/packages/runtime-browser/package.json b/packages/runtime-browser/package.json index e90be01..c1de9f5 100644 --- a/packages/runtime-browser/package.json +++ b/packages/runtime-browser/package.json @@ -1,18 +1,18 @@ { "name": "@autter/runtime-browser", - "version": "1.3.3", + "version": "1.4.0", "description": "Tiny, dependency-free browser error and usage tracker for Autter Runtime (<5 KB gzipped)", "license": "MIT", "type": "module", "sideEffects": false, - "main": "./dist/index.cjs", + "main": "./dist/index.js", "module": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js", - "require": "./dist/index.cjs" + "default": "./dist/index.js" } }, "files": [ diff --git a/packages/runtime-browser/src/index.ts b/packages/runtime-browser/src/index.ts index 8f84a20..5fe280f 100644 --- a/packages/runtime-browser/src/index.ts +++ b/packages/runtime-browser/src/index.ts @@ -141,6 +141,83 @@ export function redactContext( return out; } +// Coded errors: the same CODE_PATTERN as runtime-core and the ingester. A +// value that doesn't match (e.g. "ECONNRESET") is simply not sent, and the +// error groups by message as before. +const CODE_RE = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$/; +const REQUEST_ID_RE = /^[\w.-]{8,128}$/; + +/** + * Declared error fields duck-typed off ANY thrown value — RuntimeError, + * `autterErrorFromResponse` results, or an app's own error class with a + * `code` property — as `autter.error.*` / `autter.request.id` context keys. + */ +function errorContext(error: unknown): Record { + const out: Record = {}; + if (!error || typeof error !== "object") return out; + const e = error as Record; + if (typeof e.code === "string" && e.code.length <= 80 && CODE_RE.test(e.code)) { + out["autter.error.code"] = e.code; + } + if (typeof e.why === "string") out["autter.error.why"] = e.why.slice(0, 1000); + if (typeof e.fix === "string") out["autter.error.fix"] = e.fix.slice(0, 1000); + if (typeof e.link === "string" && e.link.length <= 500 && /^https?:\/\//.test(e.link)) { + out["autter.error.link"] = e.link; + } + if (e.expected === true) out["autter.error.expected"] = true; + if (typeof e.requestId === "string" && REQUEST_ID_RE.test(e.requestId)) { + out["autter.request.id"] = e.requestId; + } + return out; +} + +function addContext(event: BrowserEvent, extra: Record): void { + if (Object.keys(extra).length > 0) event.context = { ...(event.context || {}), ...extra }; +} + +/** `x-request-id` of a failed response → `autter.request.id`, linking the + * browser failure to the server's request summary. Cross-origin APIs must + * list the header in Access-Control-Expose-Headers for the page to see it. */ +function addRequestId(event: BrowserEvent, id: string | null | undefined): void { + if (id && REQUEST_ID_RE.test(id)) addContext(event, { "autter.request.id": id }); +} + +/** + * Turn a failed `fetch` Response into an Error carrying the server's + * declared fields. Reads `{ error: { message, code, why, fix, link, + * requestId } }` (the `autterErrorResponse` / `toClientError` body) from a + * clone, so the caller can still read the original body; anything else + * falls back to the status text. Pass the result to `captureException` (or + * throw it) and the error groups by its code. + */ +export async function autterErrorFromResponse(response: Response): Promise { + let body: Record = {}; + try { + const parsed = (await response.clone().json())?.error; + body = typeof parsed === "string" ? { message: parsed } : parsed && typeof parsed === "object" ? parsed : {}; + } catch { + /* Not JSON, or the body was already read. */ + } + const error = new Error( + typeof body.message === "string" && body.message + ? body.message + : response.statusText || `Request failed with status ${response.status}`, + ) as Error & Record; + error.name = "HttpResponseError"; + error.status = response.status; + for (const key of ["code", "why", "fix", "link", "requestId"]) { + if (typeof body[key] === "string") error[key] = body[key]; + } + if (!error.requestId) { + try { + error.requestId = response.headers.get("x-request-id") || undefined; + } catch { + /* No headers. */ + } + } + return error; +} + function route(): string { try { return location.pathname; @@ -407,7 +484,8 @@ export function captureException( event.errorType = error.name; if (error.stack) event.stack = String(error.stack).slice(0, 32000); } - if (context) event.context = { ...(event.context || {}), ...context }; + // Explicit context wins over the duck-typed declared fields. + addContext(event, { ...errorContext(error), ...context }); enqueue(event, true); } @@ -496,6 +574,7 @@ export function initAutterBrowser(options: AutterBrowserOptions): void { event.name = path; event.errorType = "HttpRequestError"; event.durationMs = Math.min(120000, Math.round(performance.now() - started)); + try { addRequestId(event, response.headers.get("x-request-id")); } catch { /* No headers. */ } enqueue(event, true); } return response; @@ -525,19 +604,24 @@ export function initAutterBrowser(options: AutterBrowserOptions): void { const started = performance.now(); const path = (() => { try { return new URL(target, location.href).pathname; } catch { return ""; } })(); let recorded = false; - const record = (message: string, errorType: string) => { + const record = (message: string, errorType: string, requestId?: string | null) => { if (recorded) return; recorded = true; const event = baseEvent("request_failure", message); event.name = path; event.errorType = errorType; event.durationMs = Math.min(120000, Math.round(performance.now() - started)); + addRequestId(event, requestId); enqueue(event, true); }; const onError = () => record("Request failed", "NetworkError"); const onTimeout = () => record("Request timed out", "NetworkError"); this.addEventListener("loadend", () => { - if (this.status >= 500) record(`Request returned ${this.status}`, "HttpRequestError"); + if (this.status >= 500) { + let requestId: string | null = null; + try { requestId = this.getResponseHeader("x-request-id"); } catch { /* Not exposed. */ } + record(`Request returned ${this.status}`, "HttpRequestError", requestId); + } this.removeEventListener("error", onError); this.removeEventListener("timeout", onTimeout); }, { once: true }); @@ -575,6 +659,7 @@ export function initAutterBrowser(options: AutterBrowserOptions): void { if (label) e.filename = label; } else { e.errorType = event.error instanceof Error ? event.error.name : "Error"; + addContext(e, errorContext(event.error)); if (event.error instanceof Error && event.error.stack) { e.stack = String(event.error.stack).slice(0, 32000); } @@ -626,6 +711,7 @@ export function initAutterBrowser(options: AutterBrowserOptions): void { "unhandled_rejection", isError ? reason.message : String(reason), ); + addContext(e, errorContext(reason)); if (isError) { e.errorType = reason.name; if (reason.stack) e.stack = String(reason.stack).slice(0, 32000); diff --git a/packages/runtime-browser/test/coded-errors.test.mjs b/packages/runtime-browser/test/coded-errors.test.mjs new file mode 100644 index 0000000..f384073 --- /dev/null +++ b/packages/runtime-browser/test/coded-errors.test.mjs @@ -0,0 +1,120 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { autterErrorFromResponse, captureException, flush, initAutterBrowser } from "../dist/index.js"; + +class FakeXHR { + status = 0; + headers = {}; + listeners = new Map(); + open(_method, _url) {} + send() {} + getResponseHeader(name) { + return this.headers[name] ?? null; + } + addEventListener(name, callback) { + this.listeners.set(name, callback); + } + removeEventListener(name, callback) { + if (this.listeners.get(name) === callback) this.listeners.delete(name); + } + emit(name) { + this.listeners.get(name)?.(); + } +} + +const sent = []; +let nextResponse; +globalThis.window = { fetch: async () => nextResponse, addEventListener() {} }; +globalThis.document = { addEventListener() {}, visibilityState: "visible" }; +globalThis.location = { href: "https://app.example.test/", pathname: "/checkout" }; +globalThis.XMLHttpRequest = FakeXHR; +Object.defineProperty(globalThis, "navigator", { configurable: true, value: { + sendBeacon(_url, body) { sent.push(body); return true; }, +} }); +initAutterBrowser({ endpoint: "https://collector.example.test/v1/browser", service: "web", sessionTracking: false, captureTimings: false }); + +async function drain() { + flush(); + const batch = sent.splice(0, sent.length); + const events = []; + for (const body of batch) events.push(...JSON.parse(await body.text()).events); + return events; +} + +test("captureException duck-types declared fields from any error", async () => { + const coded = Object.assign(new Error("Payment declined"), { + code: "billing.declined", + why: "The issuer rejected the charge", + fix: "Ask for another card", + link: "https://docs.example.com/payments#declined", + expected: true, + requestId: "req_abcdefgh12", + }); + captureException(coded, { step: "pay" }); + // Node-style codes and unsafe links are not declared codes. + captureException(Object.assign(new Error("socket"), { code: "ECONNRESET", link: "javascript:alert(1)", expected: "yes" })); + captureException({ code: "plain.object_error" }); + const [first, second, third] = await drain(); + assert.equal(first.context["autter.error.code"], "billing.declined"); + assert.equal(first.context["autter.error.why"], "The issuer rejected the charge"); + assert.equal(first.context["autter.error.fix"], "Ask for another card"); + assert.equal(first.context["autter.error.link"], "https://docs.example.com/payments#declined"); + assert.equal(first.context["autter.error.expected"], true); + assert.equal(first.context["autter.request.id"], "req_abcdefgh12"); + assert.equal(first.context.step, "pay"); + for (const key of ["autter.error.code", "autter.error.link", "autter.error.expected"]) { + assert.equal(second.context?.[key], undefined, key); + } + assert.equal(third.context["autter.error.code"], "plain.object_error"); +}); + +test("autterErrorFromResponse reads the JSON error body and falls back to status text", async () => { + const body = { error: { message: "Plan free limit reached", code: "billing.limit", why: "Quota used", fix: "Upgrade", link: "https://docs.example.com/limits", requestId: "req_0000000001" } }; + const response = new Response(JSON.stringify(body), { status: 429, statusText: "Too Many Requests", headers: { "content-type": "application/json" } }); + const error = await autterErrorFromResponse(response); + assert.ok(error instanceof Error); + assert.equal(error.message, "Plan free limit reached"); + assert.equal(error.code, "billing.limit"); + assert.equal(error.why, "Quota used"); + assert.equal(error.fix, "Upgrade"); + assert.equal(error.link, "https://docs.example.com/limits"); + assert.equal(error.requestId, "req_0000000001"); + assert.equal(error.status, 429); + // The caller can still read the original body. + assert.deepEqual(await response.json(), body); + + const html = await autterErrorFromResponse(new Response("oops", { status: 502, statusText: "Bad Gateway", headers: { "x-request-id": "req_fromheader1" } })); + assert.equal(html.message, "Bad Gateway"); + assert.equal(html.code, undefined); + assert.equal(html.requestId, "req_fromheader1"); + + const bare = await autterErrorFromResponse(new Response(null, { status: 500 })); + assert.equal(bare.message, "Request failed with status 500"); + + captureException(error); + const [event] = await drain(); + assert.equal(event.errorType, "HttpResponseError"); + assert.equal(event.context["autter.error.code"], "billing.limit"); + assert.equal(event.context["autter.request.id"], "req_0000000001"); +}); + +test("failed fetch and XHR responses carry x-request-id as autter.request.id", async () => { + nextResponse = new Response("down", { status: 503, headers: { "x-request-id": "0f6c1d2e-aaaa-bbbb-cccc-000000000001" } }); + await window.fetch("/api/orders?id=1"); + nextResponse = new Response("down", { status: 500, headers: { "x-request-id": "bad id with spaces" } }); + await window.fetch("/api/other"); + + const xhr = new XMLHttpRequest(); + xhr.open("POST", "/api/pay"); + xhr.send(); + xhr.status = 502; + xhr.headers["x-request-id"] = "req_xhr0000001"; + xhr.emit("loadend"); + + const events = await drain(); + assert.deepEqual(events.map((event) => [event.type, event.name, event.context?.["autter.request.id"]]), [ + ["request_failure", "/api/orders", "0f6c1d2e-aaaa-bbbb-cccc-000000000001"], + ["request_failure", "/api/other", undefined], + ["request_failure", "/api/pay", "req_xhr0000001"], + ]); +}); From 8ffda4f0b739b8c96d0e5cdef35348463f12ec7d Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Thu, 8 Oct 2026 00:08:04 +0530 Subject: [PATCH 03/15] feat(sdk): request summaries, coded errors, carriers and @autter/runtime-edge Adds the private runtime-core package (coded errors, record building, OTLP logs encoding, redaction) bundled into runtime-node 1.5.0 and the new @autter/runtime-edge 1.0.0. runtime-node gains request summaries (autterRequests, autterFastify, withRuntimeRequest), runtimeContext, inline messages, RuntimeError/defineRuntimeErrors/autterErrorResponse, fork/carrier, AI rollup, waitUntil, enrichers, sinks, a /testing subpath and initAutterLogging. 1.4.0 output is golden-tested. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 4 +- .gitignore | 3 + CHANGELOG.md | 36 + README.md | 59 +- docs/GETTING-STARTED.md | 51 +- docs/INTEGRATIONS.md | 48 +- docs/OPERATION-LOGGING.md | 53 +- docs/PLAN.md | 40 +- docs/REQUESTS-AND-ERRORS.md | 357 +++++++ examples/edge-worker/README.md | 16 + examples/edge-worker/package.json | 14 + examples/edge-worker/src/index.js | 40 + examples/edge-worker/wrangler.toml | 9 + examples/express-app/package.json | 2 +- examples/express-app/server.js | 47 +- package-lock.json | 58 +- package.json | 5 +- .../fixtures/code-fingerprints.json | 6 + packages/runtime-core/package.json | 26 + packages/runtime-core/src/attributes.ts | 13 + packages/runtime-core/src/carrier.ts | 85 ++ packages/runtime-core/src/context.ts | 162 +++ packages/runtime-core/src/errors.ts | 358 +++++++ packages/runtime-core/src/fingerprint-code.ts | 27 + packages/runtime-core/src/index.ts | 91 ++ packages/runtime-core/src/otlp-logs.ts | 80 ++ packages/runtime-core/src/record.ts | 194 ++++ packages/runtime-core/src/redact.ts | 409 ++++++++ packages/runtime-core/src/request.ts | 95 ++ packages/runtime-core/src/user-agent.ts | 55 + packages/runtime-core/test/core.test.mjs | 233 +++++ packages/runtime-core/tsconfig.json | 13 + packages/runtime-edge/README.md | 117 +++ packages/runtime-edge/package.json | 54 + packages/runtime-edge/src/index.ts | 477 +++++++++ packages/runtime-edge/test/edge.test.mjs | 143 +++ packages/runtime-edge/tsconfig.json | 13 + packages/runtime-edge/tsup.config.ts | 14 + packages/runtime-next/README.md | 48 + packages/runtime-next/package.json | 17 +- packages/runtime-next/src/client.ts | 1 + packages/runtime-next/src/edge.ts | 14 + packages/runtime-next/src/server.ts | 107 +- packages/runtime-next/test/next.test.mjs | 65 ++ packages/runtime-next/tsup.config.ts | 13 +- packages/runtime-node/README.md | 43 + packages/runtime-node/package.json | 16 +- packages/runtime-node/src/enrichers.ts | 110 ++ packages/runtime-node/src/errors.ts | 20 + packages/runtime-node/src/index.ts | 40 + packages/runtime-node/src/llm-instrument.ts | 35 +- packages/runtime-node/src/logger.ts | 990 +++++++++++------- packages/runtime-node/src/logging-only.ts | 184 ++++ packages/runtime-node/src/redact.ts | 415 +------- packages/runtime-node/src/requests.ts | 613 +++++++++++ packages/runtime-node/src/server.ts | 155 ++- packages/runtime-node/src/sinks.ts | 529 ++++++++++ packages/runtime-node/src/testing.ts | 162 +++ .../test/fixtures/golden-1.4.0-logs.json | 521 +++++++++ .../test/fixtures/golden-scenario.mjs | 136 +++ .../test/fixtures/write-golden.mjs | 17 + packages/runtime-node/test/golden.test.mjs | 34 + packages/runtime-node/test/hook-mode.test.mjs | 53 + .../runtime-node/test/logging-only.test.mjs | 68 ++ .../runtime-node/test/operations.test.mjs | 217 ++++ packages/runtime-node/test/requests.test.mjs | 308 ++++++ packages/runtime-node/test/sinks.test.mjs | 89 ++ packages/runtime-node/tsup.config.ts | 15 + 69 files changed, 7695 insertions(+), 850 deletions(-) create mode 100644 docs/REQUESTS-AND-ERRORS.md create mode 100644 examples/edge-worker/README.md create mode 100644 examples/edge-worker/package.json create mode 100644 examples/edge-worker/src/index.js create mode 100644 examples/edge-worker/wrangler.toml create mode 100644 packages/runtime-core/fixtures/code-fingerprints.json create mode 100644 packages/runtime-core/package.json create mode 100644 packages/runtime-core/src/attributes.ts create mode 100644 packages/runtime-core/src/carrier.ts create mode 100644 packages/runtime-core/src/context.ts create mode 100644 packages/runtime-core/src/errors.ts create mode 100644 packages/runtime-core/src/fingerprint-code.ts create mode 100644 packages/runtime-core/src/index.ts create mode 100644 packages/runtime-core/src/otlp-logs.ts create mode 100644 packages/runtime-core/src/record.ts create mode 100644 packages/runtime-core/src/redact.ts create mode 100644 packages/runtime-core/src/request.ts create mode 100644 packages/runtime-core/src/user-agent.ts create mode 100644 packages/runtime-core/test/core.test.mjs create mode 100644 packages/runtime-core/tsconfig.json create mode 100644 packages/runtime-edge/README.md create mode 100644 packages/runtime-edge/package.json create mode 100644 packages/runtime-edge/src/index.ts create mode 100644 packages/runtime-edge/test/edge.test.mjs create mode 100644 packages/runtime-edge/tsconfig.json create mode 100644 packages/runtime-edge/tsup.config.ts create mode 100644 packages/runtime-next/src/edge.ts create mode 100644 packages/runtime-next/test/next.test.mjs create mode 100644 packages/runtime-node/src/enrichers.ts create mode 100644 packages/runtime-node/src/errors.ts create mode 100644 packages/runtime-node/src/logging-only.ts create mode 100644 packages/runtime-node/src/requests.ts create mode 100644 packages/runtime-node/src/sinks.ts create mode 100644 packages/runtime-node/src/testing.ts create mode 100644 packages/runtime-node/test/fixtures/golden-1.4.0-logs.json create mode 100644 packages/runtime-node/test/fixtures/golden-scenario.mjs create mode 100644 packages/runtime-node/test/fixtures/write-golden.mjs create mode 100644 packages/runtime-node/test/golden.test.mjs create mode 100644 packages/runtime-node/test/hook-mode.test.mjs create mode 100644 packages/runtime-node/test/logging-only.test.mjs create mode 100644 packages/runtime-node/test/operations.test.mjs create mode 100644 packages/runtime-node/test/requests.test.mjs create mode 100644 packages/runtime-node/test/sinks.test.mjs create mode 100644 packages/runtime-node/tsup.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7928ece..bc38adc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,10 @@ jobs: node-version: 22 - run: npm install - run: npm run build + - run: npm run test -w @autter/runtime-core - run: npm run test -w @autter/runtime-node + - run: npm run test -w @autter/runtime-edge + - run: npm run test -w @autter/runtime-next - run: npm run test -w @autter/runtime-browser - run: npm run test -w @autter/otlp-ingester - run: npm run size -w @autter/runtime-browser diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a0bc125..dfa96a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,9 @@ jobs: - run: npm run size -w @autter/runtime-browser - name: Publish packages (skips versions that already exist) run: | - for pkg in runtime-browser runtime-node runtime-next otlp-ingester; do + # runtime-core is private (bundled into node + edge) and never published. + # Dependencies first: runtime-next depends on node, edge and browser. + for pkg in runtime-browser runtime-node runtime-edge runtime-next otlp-ingester; do name=$(node -p "require('./packages/$pkg/package.json').name") version=$(node -p "require('./packages/$pkg/package.json').version") if npm view "$name@$version" version >/dev/null 2>&1; then diff --git a/.gitignore b/.gitignore index f87cba4..9e02ec0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ examples/express-app/public/sdk/ .env.* !.env.example .DS_Store + +# Local runtime files written by fileSink in development +.autter/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c0cd1b..d908fd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Changelog +## [1.5.0] - 2026-10-08 + +Packages: `otlp-ingester` 1.5.0, `runtime-node` 1.5.0, `runtime-next` 1.5.0, `runtime-browser` 1.4.0, new `runtime-edge` 1.0.0. Additive: 1.4.0 code keeps working unchanged (golden-tested). + +### Features + +- Request summaries: `autterRequests` (Express/Connect), `autterFastify` and `withRuntimeRequest` emit one summary per request with route, status, duration, outcome and a request id. `x-request-id` is honoured or generated, echoed (CORS-exposed) and stamped on spans and records. Summaries are always kept; `ignore` globs skip health/metrics routes (`runtime-node`, `runtime-next`). +- `runtimeContext` (`set`, `outcome`, `debug/info/warn/error`, `id`, `requestId`, `fork`, `carrier`) and `runInBackground`. Debug/info messages inside an operation fold into its summary (`autter.operation.logs`, max 50) and summaries carry `kind` and `autter.operation.level` (`runtime-node`). +- Coded errors: `RuntimeError`, `defineRuntimeErrors`, `isRuntimeErrorLike`, `toClientError` and `autterErrorResponse`. `captureException` reads `code/why/fix/link/status/expected` from any error, records cause chains, keeps `internal` on the span only, and treats `expected` failures as `degraded` (`runtime-node`). +- Code-based grouping: errors with a valid `autter.error.code` get the source-independent `code-v1` fingerprint; uncoded errors keep `message-v1` byte-for-byte (`otlp-ingester`). +- Cross-process carriers (`withRuntimeOperation(name, fn, attrs, { from, waitUntil })`) with OTel span links, and per-operation AI usage rollup (`autter.operation.ai`) (`runtime-node`). +- `initAutterLogging` runs requests, operations and coded errors without starting NodeSDK, for apps that already run their own OpenTelemetry SDK. Its error records (`autter.capture.mode=log`) are promoted to occurrences by `/v1/logs`, deduplicated by trace id (`runtime-node`, `otlp-ingester`). +- Enrichers (`enrichUserAgent`, `enrichRequestSize`, `enrichEdgeGeo`, `enrichDeployment`), sinks (`otlpSink`, `consoleSink`, `fileSink` writing `.autter/runtime/*.jsonl` in development) and the `@autter/runtime-node/testing` subpath (`runtime-node`). +- New `@autter/runtime-edge` 1.0.0: `withAutter` for Cloudflare Workers, Vercel Edge, Deno and Bun with zero dependencies; also exposed as `@autter/runtime-next/edge`. +- `runtime_logs` gains `kind`, `request_id`, `route`, `status_code`, `error_code`, `ai_cost_usd`, `ai_calls` (migration 0012) and feeds the `runtime_request_1m` rollup (migration 0014). `runtime_error_occurrences` gains `error_code`, `error_why`, `error_fix`, `error_link`, `expected`, `request_id`, `fingerprint_scheme` (migration 0013) (`otlp-ingester`). +- Sink payload (still `version: 1`) adds optional `errorCode`, `why`, `fix`, `link`, `expected`, `requestId`, `traceId`, `route`, `method`, `statusCode` and `fingerprintScheme`; absent values are omitted (`otlp-ingester`). +- New `LOG_TTL_DAYS` (default 14) applied idempotently to `runtime_logs` at boot (`otlp-ingester`). +- Browser: `captureException` and the global handlers send `code/why/fix/link/expected/requestId`; new `autterErrorFromResponse(response)`; failed fetch/XHR 5xx responses carry `x-request-id` (`runtime-browser`). +- Experimental zero-code `logging.requests` hook mode, off by default (`runtime-node`). + +### Changes + +- Console output is a pretty tree outside production; production keeps JSON lines (`runtime-node`). +- Debug/info logged inside an operation are folded into its summary instead of separate records; `logging.inline: false` restores 1.4.0 behaviour (`runtime-node`). +- `cache_read_tokens` / `cache_creation_tokens` counts are no longer masked as tokens by redaction (`runtime-node`, `otlp-ingester`). + +### Fixes + +- `runtime-browser` no longer declares a `./dist/index.cjs` entry that the ESM-only build never produced. +- `runtime-next` builds no longer race on `dist/` cleanup and drop `client.d.ts` / `edge.d.ts`. + +### Upgrade + +- Deploy otlp-ingester 1.5.0 first (migrations 0012–0014 run at boot), then SDKs. Older ingesters accept 1.5.0 SDK traffic but ignore the new fields. +- The `runtime_request_1m` rollup counts every insert, so it is approximate when exporters retry a batch. + ## [1.3.1] - 2026-09-08 ### Fixes diff --git a/README.md b/README.md index bc86342..c5df166 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,11 @@ and writes them to ClickHouse in a compact, per-repo data model. flowchart TD A["@autter/runtime-browser (tiny tracker)"] --> B["Same-origin relay (@autter/runtime-node)"] B --> D["otlp-ingester /v1/browser (JSON)"] - C["Server OpenTelemetry"] --> E["otlp-ingester /v1/traces + /v1/metrics (OTLP)"] + C["Server OpenTelemetry + request summaries"] --> E["otlp-ingester /v1/traces + /v1/metrics + /v1/logs (OTLP)"] + I["@autter/runtime-edge (Workers, Vercel Edge, Deno, Bun)"] --> E D --> F["Normaliser + fingerprinter"] E --> F - F --> G["ClickHouse (occurrences, spans, usage rollups)"] + F --> G["ClickHouse (occurrences, spans, logs, usage rollups)"] F --> H["Optional sink webhook → issue grouping"] ``` @@ -30,8 +31,17 @@ Connect Sentry, PostHog, Grafana/Loki, Datadog or webhooks in the Autter platfor npm install @autter/runtime-browser # frontend (React, Vue, any SPA, static sites) npm install @autter/runtime-node # backend (Express, Fastify, Koa, Nest, plain Node) npm install @autter/runtime-next # Next.js (both halves in one package) +npm install @autter/runtime-edge # Cloudflare Workers, Vercel Edge, Deno, Bun (zero deps) ``` +| Package | Version | Use it for | +| --- | --- | --- | +| `@autter/runtime-browser` | 1.4.0 | Browser errors + usage, coded-error duck typing, request-id capture | +| `@autter/runtime-node` | 1.5.0 | Node servers: OTel tracing, request summaries, coded errors, operations | +| `@autter/runtime-next` | 1.5.0 | Next.js: server + client + `/edge` for `middleware.ts` | +| `@autter/runtime-edge` | 1.0.0 | Fetch-only runtimes: request summaries and coded errors over `/v1/logs` | +| `@autter/otlp-ingester` | 1.5.0 | Self-hosted ingest — upgrade it **before** the SDKs | + Prefer to have an AI agent set it up for you? Install the companion agent skills — they inventory your repo and wire up Autter Runtime for whatever language/framework each service uses, npm packages or not: @@ -78,6 +88,26 @@ initAutterServer({ }); ``` +**Requests and coded errors** (1.5.0) — one summary per request, always +kept, with a request id echoed to the client; one issue per error code: + +```js +const { autterRequests, autterErrorResponse, defineRuntimeErrors, runtimeContext } = + require("@autter/runtime-node"); + +const billing = defineRuntimeErrors("billing", { + declined: { status: 402, message: "Payment declined", expected: true, fix: "Try another card" }, +}); +app.use(autterRequests({ ignore: ["/healthz"] })); +app.post("/checkout", (req, res, next) => { + runtimeContext.set({ cart: { items: 3 } }); + next(billing.declined()); // → 402 { error: { message, code, fix, requestId } } +}); +app.use(autterErrorResponse()); +``` + +See [Requests, coded errors and background work](docs/REQUESTS-AND-ERRORS.md). + **LLM calls** — initialised with the server tracker, recorded at 100% (model, tokens, latency, USD cost). One line per client: @@ -100,14 +130,19 @@ Full walkthrough (keys, relay setup, Next.js, verification): | Package | Status | Description | | --- | --- | --- | -| [`@autter/runtime-browser`](packages/runtime-browser) | **v1.0** | Zero-dependency browser error + usage tracker (~1 KB brotlied) | -| [`@autter/runtime-node`](packages/runtime-node) | **v1.0** | Same-origin relay handler + curated OTel server tracker | -| [`@autter/runtime-next`](packages/runtime-next) | **v1.0** | One-command Next.js integration (relay route + error boundary) | -| [`@autter/otlp-ingester`](packages/otlp-ingester) | **v1.0** | Self-hostable ingest service: OTLP/HTTP (protobuf + JSON) traces + metrics, browser payloads → ClickHouse | +| [`@autter/runtime-browser`](packages/runtime-browser) | **v1.4** | Zero-dependency browser error + usage tracker (<5 KB gzipped) | +| [`@autter/runtime-node`](packages/runtime-node) | **v1.5** | Curated OTel server tracker, request summaries, coded errors, operation logging, same-origin relay | +| [`@autter/runtime-next`](packages/runtime-next) | **v1.5** | One-command Next.js integration (server, client, relay route, error boundary, edge middleware) | +| [`@autter/runtime-edge`](packages/runtime-edge) | **v1.0** | Zero-dependency edge SDK: request summaries and coded errors for Workers, Vercel Edge, Deno, Bun | +| [`@autter/otlp-ingester`](packages/otlp-ingester) | **v1.5** | Self-hostable ingest service: OTLP/HTTP (protobuf + JSON) traces, metrics, logs, browser payloads → ClickHouse | + +`packages/runtime-core` is private: shared code bundled into runtime-node and +runtime-edge at build time, never published. Runnable demo: [`examples/express-app`](examples/express-app) — browser -tracker → relay → ingester and OTel server tracker, against a compose-run -ClickHouse. +tracker → relay → ingester, OTel server tracker, request summaries and coded +errors, against a compose-run ClickHouse. Edge: +[`examples/edge-worker`](examples/edge-worker) (Cloudflare Worker). ## Supported stacks @@ -119,6 +154,7 @@ For endpoint latency detection and slow-request retention, see [endpoint regress | React/SPA with a backend | `@autter/runtime-browser` → relay | none in browser; server key in relay | | Next.js | `@autter/runtime-next` | server key | | Node (Express, Fastify, Koa, Nest) | `@autter/runtime-node` | server key | +| Cloudflare Workers, Vercel Edge, Deno, Bun | `@autter/runtime-edge` | server key (secret binding) | | Go, Rust, Python, Java, .NET, … | any OTel SDK → OTLP/HTTP (protobuf **or** JSON) | server key | Per-stack setup snippets: [`docs/INTEGRATIONS.md`](docs/INTEGRATIONS.md). @@ -130,7 +166,7 @@ Two credential types keep the frontend and backend cleanly separated: | | Server key (`autter_rt_…`) | Client key (`autter_rtc_…`) | | --- | --- | --- | | Secrecy | **secret** — backend env vars only | **publishable** — safe in frontend bundles | -| Can send | OTLP traces/metrics + browser events | browser events only | +| Can send | OTLP traces/metrics/logs + browser events | browser events only | | Protection | rate limits | origin allow-list + tighter rate limits, write-only | When your app has a backend, prefer the **relay**: the browser posts to your @@ -144,6 +180,7 @@ all, and ad-blockers can't tell it apart from your own API traffic. - [Using Autter Runtime **without npm**](docs/WITHOUT-NPM.md) — any OTel SDK, an OTel Collector, or plain HTTP from any language - [Architecture & data model](docs/ARCHITECTURE.md) - [Operation logging and diagnostic context](docs/OPERATION-LOGGING.md) +- [Requests, coded errors and background work](docs/REQUESTS-AND-ERRORS.md) - [Continuous detection, profiles, and outcomes](docs/CONTINUOUS-DETECTION.md) - [Roadmap](docs/PLAN.md) · [Releasing](docs/RELEASING.md) @@ -196,7 +233,9 @@ new OTLPTraceExporter({ occurrences are always kept (14-day TTL); traces containing an error are retained in full (the Node SDK tail-retains them), so every issue keeps the trace that explains it; healthy traces are expected to be sampled upstream - (0.5–1%); usage is stored as 1-minute rollups (90 days). + (0.5–1%); usage is stored as 1-minute rollups (90 days). Request and + operation summaries are one bounded record per unit of work and are always + kept (`LOG_TTL_DAYS`, default 14). - **Per-repo analysis.** Every row is keyed by `org_id` + `repository_id`. - **Privacy by construction.** No cookies, no DOM, no request/response bodies, no emails, no full URLs with query strings. diff --git a/docs/GETTING-STARTED.md b/docs/GETTING-STARTED.md index b19d872..483fd64 100644 --- a/docs/GETTING-STARTED.md +++ b/docs/GETTING-STARTED.md @@ -30,7 +30,7 @@ you from zero to seeing data in ClickHouse. | --- | --- | --- | | Looks like | `autter_rt_…` | `autter_rtc_…` | | Secrecy | **secret** — env vars only, never in a browser bundle | **publishable** — safe to ship in frontend JS | -| Can send | OTLP traces + metrics, browser events | browser events only (`/v1/browser`) | +| Can send | OTLP traces + metrics + logs, browser events | browser events only (`/v1/browser`) | | Extra protection | — | per-key origin allow-list, tighter rate limit | **Two ways to send browser events:** @@ -51,6 +51,8 @@ you from zero to seeing data in ClickHouse. | Handled errors | `captureException(err)` | `captureException(err)` | | Usage | session pings + `trackEvent()` | request counts/durations per route (automatic) | | Traces | — (by design; no OTel in the browser) | ~1% sampled, plus **every erroring trace kept in full** | +| Request summaries | request id on failed fetch/XHR (links to the server request) | one record per request with route, status, outcome, context, inline messages — **always kept** (`autterRequests`, 1.5.0) | +| Coded errors | `code`/`why`/`fix` duck-typed by `captureException` | `defineRuntimeErrors` / any error with `code` — **one issue per code** | | LLM usage & cost | — | `withLlmCall()` / Vercel AI SDK telemetry / GenAI semconv — always 100% (model, tokens, cost) | **What is never sent:** cookies, DOM content, form values, request/response @@ -146,6 +148,37 @@ ESM-only app? Use `--import` plus OTel's loader hook (see the enriched with `@opentelemetry/instrumentation-express` via the `instrumentations` option. +### Request summaries and coded errors (1.5.0) + +Requires ingester 1.5.0. One record per request, always kept, with the request +id echoed to the client: + +```js +const { autterRequests, autterErrorResponse, defineRuntimeErrors, runtimeContext } = + require("@autter/runtime-node"); + +const billingErrors = defineRuntimeErrors("billing", { + declined: { status: 402, message: "Payment declined", expected: true, + fix: "Ask the customer for another card" }, +}); + +app.use(autterRequests({ ignore: ["/healthz"] })); // before your routes +app.post("/checkout", async (req, res) => { + runtimeContext.set({ cart: { items: req.body.items.length } }); + if (!(await charge(req.body))) throw billingErrors.declined(); // Express 5; next(err) on 4 + res.json({ ok: true }); +}); +app.use(autterErrorResponse()); // after your routes +``` + +Errors with the same code become one issue; `expected` ones are recorded but +never open incidents. Already running your own OpenTelemetry SDK? Use +`initAutterLogging` instead of `initAutterServer`. In development, records are +also written to `.autter/runtime/*.jsonl` (add `.autter/` to `.gitignore`). +Everything else — `runtimeContext`, background work and queues, AI usage per +request, testing helpers, edge runtimes: +[Requests, coded errors and background work](REQUESTS-AND-ERRORS.md). + ### Track LLM usage & cost For the complete `withLlmCall` / `trackLlmCall` API, emitted `gen_ai.*` @@ -272,7 +305,12 @@ Three files and you have server tracing, browser tracking, the relay, and the error boundary — see the [`@autter/runtime-next` README](../packages/runtime-next) for the exact snippets (`instrumentation.ts`, `app/api/autter-runtime/route.ts`, and a -client component). +client component). Wrap route handlers in `withRuntimeRequest` for request +summaries; `middleware.ts` uses `@autter/runtime-next/edge`. + +Cloudflare Workers, Vercel Edge, Deno or Bun without Next.js: +`npm install @autter/runtime-edge` and wrap the fetch handler in `withAutter` +(see [the package README](../packages/runtime-edge)). ## 6. Other languages @@ -315,6 +353,12 @@ FROM autter_runtime.runtime_metrics_1m WHERE bucket_at > now() - INTERVAL 1 HOUR GROUP BY service, route; +-- Request summaries (runtime-node 1.5.0 + autterRequests) +SELECT occurred_at, operation, route, status_code, outcome, request_id, error_code +FROM autter_runtime.runtime_logs +WHERE kind = 'request' +ORDER BY occurred_at DESC LIMIT 10; + -- LLM calls (after emitLlmSelftestTrace() or a real traced model call) SELECT service, provider, model, input_tokens, output_tokens, cost_usd, cost_source, status, started_at @@ -332,6 +376,9 @@ clickhouse-client --password dev`. - [ ] Client keys have `allowedOrigins` set to your exact app origins. - [ ] `release` is wired to your git SHA in **both** frontend and backend — it's what powers regression detection ("broke in release X"). +- [ ] Request summaries are always kept: `ignore` health checks and metrics + scrapes in `autterRequests`, and size `LOG_TTL_DAYS` at the ingester. +- [ ] Error codes are stable and low-cardinality — never ids or user data. - [ ] Keep trace sampling at ~1% (`traceSampleRate`) — errors are always captured regardless, and the full trace of an erroring request is retained (`retainTracesOnError`, on by default), so cheap sampling diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 1afcef7..51dd82a 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -5,12 +5,12 @@ profile and source-map uploads use the server key. | Credential | Where it lives | Can | | --- | --- | --- | -| **Server key** `autter_rt_…` | backend env vars only | send OTLP traces/metrics + relay browser events | +| **Server key** `autter_rt_…` | backend env vars / edge secret bindings only | send OTLP traces/metrics/logs + relay browser events | | **Client key** `autter_rtc_…` | frontend bundles (publishable) | send browser events only, origin-restricted | | Endpoint | Format | | --- | --- | -| `POST /v1/traces`, `POST /v1/metrics` | OTLP/HTTP — **protobuf or JSON**, gzip ok | +| `POST /v1/traces`, `POST /v1/metrics`, `POST /v1/logs` | OTLP/HTTP — **protobuf or JSON**, gzip ok | | `POST /v1/browser` | compact JSON (`@autter/runtime-browser` payload v1) | | `POST /v1/profiles` | symbolized pprof (server key only) | | `POST /v1/sourcemaps` | release-keyed source map JSON (server key only) | @@ -74,10 +74,43 @@ initAutterServer({ Relay for your frontend: `app.post("/api/autter-runtime", createBrowserRelayHandler({ apiKey }))`. ESM-only apps need OTel's loader hook (see the package README). +Request summaries and coded errors (1.5.0, ingester 1.5.0 first): + +| Framework | Wiring | +| --- | --- | +| Express / Connect | `app.use(autterRequests())` early, `app.use(autterErrorResponse())` last | +| Fastify | `app.register(autterFastify)`; use `toClientError(err, runtimeContext.requestId)` in `setErrorHandler` | +| Koa / Hono on Node / custom fetch handlers | `withRuntimeRequest(handler)` around the fetch-style handler, or `withRuntimeOperation` per request | +| NestJS (Express adapter) | `app.use(autterRequests())` in `main.ts`; an exception filter calling `toClientError` | + +App already runs its own OpenTelemetry SDK? Use `initAutterLogging` instead of +`initAutterServer` — requests, operations and coded errors without a second +NodeSDK. Details: [Requests, coded errors and background work](REQUESTS-AND-ERRORS.md). + +## Edge runtimes (Cloudflare Workers, Vercel Edge, Deno, Bun) + +```bash +npm install @autter/runtime-edge +``` + +```ts +import { withAutter } from "@autter/runtime-edge"; +export default withAutter( + (env) => ({ apiKey: env.AUTTER_RUNTIME_KEY, service: "edge-api" }), + async (request, env, ctx, rt) => new Response("ok"), +); +``` + +Zero dependencies; request summaries, request ids and coded errors over +`/v1/logs`, delivered through `ctx.waitUntil`. Needs a **server** key from a +secret binding. Next.js `middleware.ts`: import from `@autter/runtime-next/edge`. + ## Next.js `npm install @autter/runtime-next` — three files (instrumentation.ts, relay -route, client init + boundary). See the package README. +route, client init + boundary). See the package README. Route handlers: +`export const POST = withRuntimeRequest(handler)` (flush wired to `after()`); +`middleware.ts`: `@autter/runtime-next/edge`. ## Go @@ -127,6 +160,15 @@ OTEL_TRACES_SAMPLER_ARG=0.01 Errors surface as issues when spans carry `exception` events (every SDK's `record_exception` / `RecordError` does this) or `ERROR` status. +**Coded errors** (any language): add `autter.error.code` (matching +`^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$`, ≤ 80 chars) and optionally +`autter.error.why`, `autter.error.fix`, `autter.error.link` and +`autter.error.expected` (boolean) to the exception event or the span. Coded +errors group by service + code across messages and sources; `expected` errors +never open incidents. Request summaries from other languages are OTLP log +records with `autter.event.type=operation`, `autter.operation.kind=request`, +`autter.request.id`, `http.route` and `http.response.status_code`. + **Warnings** (any language): add an `autter.severity` attribute to the exception event — `"fatal" | "error" | "warning" | "info"` (default `error`; the Node SDK's crash marker `autter.unhandled: true` implies diff --git a/docs/OPERATION-LOGGING.md b/docs/OPERATION-LOGGING.md index 3f3ab4b..3ca9af3 100644 --- a/docs/OPERATION-LOGGING.md +++ b/docs/OPERATION-LOGGING.md @@ -4,12 +4,21 @@ Operation logging starts in `@autter/runtime-node` and `@autter/runtime-next` version **1.4.0**. Update the ingester to **1.4.0** before upgrading SDKs: it creates `runtime_logs` through migration `0011-runtime-logs` and accepts `/v1/logs`. +**1.5.0** (ingester 1.5.0 first) adds request summaries, `runtimeContext`, inline +messages, `kind` on every summary, coded errors, `fork`/carriers, the AI rollup, +`waitUntil`, sinks and logger-only mode — all additive; a golden test proves a +1.4.0 operation still exports the same OTLP JSON apart from the two new summary +keys. Request-level features are documented in +[REQUESTS-AND-ERRORS.md](REQUESTS-AND-ERRORS.md). + ## Node and Next.js Initialize `initAutterServer` once, before the application starts. For Next.js, use `registerAutter` in `instrumentation.ts` and import the APIs below from `@autter/runtime-next/server`. Use the Node runtime; these APIs use Node's async -context and do not run in the browser or an edge worker. +context and do not run in the browser or an edge worker (edge runtimes use +`@autter/runtime-edge`, which passes the context explicitly). Apps that run their +own OpenTelemetry SDK can use `initAutterLogging` instead of `initAutterServer`. ```ts import { @@ -39,8 +48,12 @@ await withRuntimeOperation("checkout", async (operation) => { await runtime.shutdown(); ``` -`withRuntimeOperation(name, fn, attributes?)` runs the callback in an always-recorded -process span and emits one completed operation summary. It returns the callback's +`withRuntimeOperation(name, fn, attributes?, options?)` runs the callback in an always-recorded +process span and emits one completed operation summary (`autter.operation.kind = +"operation"`; request middleware emits `"request"` summaries). `options.from` +continues work from a `runtimeContext.carrier()` (parent link, request id, and an +OTel span link to the producer trace); `options.waitUntil` receives the log flush +promise once the summary is emitted. It returns the callback's result and rethrows its original error. An unhandled callback error records an exception in the trace; the log summary is related evidence, not a second issue. @@ -62,13 +75,26 @@ the existing `autter.outcome` trace event, including the reporting call site. `pending` means the operation has not confirmed its downstream result. `createRuntimeLogger(attributes?)` creates a logger with `debug`, `info`, `warn`, -and `error` methods. `runtimeLogger` is the default instance. `error` records +and `error` methods. `runtimeLogger` is the default instance; `runtimeContext` +exposes the same methods plus `set`, `outcome`, `id`, `requestId`, `fork` and +`carrier` for the current operation from anywhere in its call tree. + +**Inline messages (1.5.0).** Inside an operation, `debug` and `info` messages are +folded into its summary as `autter.operation.logs` — `{ t, level, message, +attrs? }`, `t` in milliseconds since the operation started — instead of being +exported as separate records. `warn` and `error` are folded **and** exported. +`autter.operation.level` is the highest level seen (including a failed outcome). +Disable with `logging.inline: false` to get 1.4.0's one-record-per-message +behaviour. Outside an operation every message is its own record, as before. `error` records diagnostic context; use `captureException` for exception grouping or declare a failed operation outcome for business-failure grouping. Logs inherit the active operation ID, name, parent operation ID, and application attributes, plus the active OTel trace/span IDs. Concurrent operations keep separate contexts. -Async context does not cross a queue or process: propagate an opaque workflow -identifier in the job payload and pass it as a custom attribute in the consumer. +Async context does not cross a queue or process by itself: put +`runtimeContext.carrier()` in the job payload and pass it as +`withRuntimeOperation(name, fn, {}, { from: payload.autter })` in the consumer. +`runtimeContext.fork(name, fn)` links a child to its parent even when the child +outlives it; `runInBackground(name, fn)` does the same without awaiting. ## Collection and privacy @@ -79,9 +105,16 @@ identifier in the job payload and pass it as a custom attribute in the consumer. cannot send server logs. - Redaction applies before console output and export. Never intentionally send secrets or personal data; pattern-based redaction cannot identify every secret. -- `logging.minLevel` filters ordinary messages; completed operation summaries - are retained independently. `logging.console` defaults to true and can be - disabled without disabling export. +- `logging.minLevel` filters ordinary messages (folded or not); completed + operation summaries are retained independently. `logging.console` defaults to + true and can be disabled without disabling export. Console output is JSON + lines (as in 1.4.0) when `NODE_ENV=production` and a compact tree elsewhere; + `logging.sinks` replaces the default sinks, and `logging.file` controls the + local `.autter/runtime/*.jsonl` files (on by default only in development). +- Summaries are never sampled. Inline messages are capped at 50 per operation + (`autter.operation.logs_truncated` marks the rest), 300 characters each and + about 6 KB per operation, and are placed last in the context budget so they + never crowd out operation context. - Logs batch in memory (up to 1,000 records or 4 MiB, whichever comes first). Requests contain up to 50 records or 512 KiB. Context has depth, field, step and string limits; truncated context is marked. Each flush has a 10-second @@ -90,7 +123,7 @@ identifier in the job payload and pass it as a custom attribute in the consumer. reported. `flushRuntimeLogs()` rejects if delivery fails, and `runtimeLogStats()` reports buffered and dropped counts. This is best-effort telemetry, not durable delivery or an audit log. -- Records expire after 14 days. Successful operation counts represent captured +- Records expire after 14 days by default (`LOG_TTL_DAYS` on ingester 1.5.0+). Successful operation counts represent captured summaries, not a guarantee that every application operation was observed. ## Investigation and fixes diff --git a/docs/PLAN.md b/docs/PLAN.md index 5bb97c8..c092dab 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -113,6 +113,41 @@ initialisation in the Node SDKs: - Non-Node stacks: any OTel GenAI instrumentation works — see INTEGRATIONS.md for the sampling exemption they must add. +## Milestone 7 — Operation logging (1.4.0, shipped) + +`/v1/logs` (OTLP JSON + protobuf) → `runtime_logs`; `withRuntimeOperation`, +`runtimeLogger`, steps and outcomes in runtime-node/next 1.4.0. See +OPERATION-LOGGING.md. + +## Milestone 8 — Request wide events and coded errors (1.5.0, shipped) + +Releases: runtime-node/next 1.5.0, **new `@autter/runtime-edge` 1.0.0**, +runtime-browser 1.4.0, otlp-ingester 1.5.0 (ingester first). Built natively on +1.4.0 operations; no dependency on evlog. See REQUESTS-AND-ERRORS.md. + +- **Private `packages/runtime-core`** (never published, bundled into node and + edge): coded errors, context bounding/redaction, record building, OTLP logs + encoding, carriers, the `code-v1` fingerprint with shared test vectors. +- **Logger pipeline** `enrich → redact/bound → sinks[]`, with a golden test + proving a 1.4.0 operation still exports byte-identical OTLP JSON (apart from + the additive `kind`/`level` keys). +- **Request summaries** — `autterRequests`, `autterFastify`, + `withRuntimeRequest` (Next: `after()` wired), experimental `logging.requests` + hook mode; request ids honoured/echoed/exposed to CORS; always kept, no + sampling (`ignore` globs, field budgets and `LOG_TTL_DAYS` are the volume + knobs). +- **`runtimeContext`**, inline messages (`autter.operation.logs`, max 50), + `autter.operation.level`, `kind` on every summary. +- **Coded errors** — `RuntimeError`, `defineRuntimeErrors`, duck-typed + `captureException`, `autterErrorResponse`/`toClientError`, `expected` + business failures, span-only `internal`, cause chains; ingester groups by code. +- **Beyond the request** — `fork`, `runInBackground`, carriers with OTel links, + AI usage rollup, `waitUntil`. +- **Logger-only mode** `initAutterLogging` (no NodeSDK; promoted log + exceptions) and `@autter/runtime-edge` (Workers, Vercel Edge, Deno, Bun). +- Enrichers, `otlp`/`console`/`file` sinks (`.autter/runtime/*.jsonl` in + development), `@autter/runtime-node/testing`. + ## Later / explicitly deferred - Opt-in same-origin network tracing (`traceparent` propagation) in the @@ -120,7 +155,6 @@ initialisation in the Node SDKs: - Web Vitals, failed-request capture. - Public DSN-style endpoint for static sites (origin allow-list, aggressive rate limits). -- Logs signal (`/v1/logs`). - Full OpenTelemetry browser SDK support — only if demanded. ## Compatibility contract @@ -130,5 +164,7 @@ initialisation in the Node SDKs: | `/v1/traces`, `/v1/metrics` OTLP/HTTP | OTLP spec-stable | | `/v1/browser` payload (`version: 1`) | additive-only changes | | ClickHouse table schemas | additive-only; TTLs configurable via env | -| Sink webhook payload (`version: 1`) | additive-only (`llmCalls`, `batchId`) | +| `/v1/logs` OTLP/HTTP | OTLP spec-stable; Autter attributes (`autter.operation.*`, `autter.request.id`, `autter.error.*`) additive-only | +| Sink webhook payload (`version: 1`) | additive-only (`llmCalls`, `batchId`, coded-error/request fields) | +| `runtimeContext.carrier()` (`v: 1`) | additive-only | | Sink webhook delivery | at-least-once; dedupe on `batchId`/`occurrenceId` | diff --git a/docs/REQUESTS-AND-ERRORS.md b/docs/REQUESTS-AND-ERRORS.md new file mode 100644 index 0000000..8349146 --- /dev/null +++ b/docs/REQUESTS-AND-ERRORS.md @@ -0,0 +1,357 @@ +# Requests, coded errors and background work + +Request wide events and coded errors start in `@autter/runtime-node` and +`@autter/runtime-next` **1.5.0** and the new `@autter/runtime-edge` **1.0.0**. +Upgrade the ingester to **1.5.0** first: it stores the new request columns +(`kind`, `request_id`, `route`, `status_code`, `error_code`, AI totals), groups +coded errors by code, and promotes log-captured exceptions to issues. Older +ingesters accept the records but ignore the new fields. + +Everything here is additive. 1.4.0 code keeps working unchanged; see +[OPERATION-LOGGING.md](OPERATION-LOGGING.md) for operations, steps and limits. + +## Request summaries + +One record per HTTP request, emitted when the response finishes (or the +connection closes). It carries the method, route template, status, outcome, +duration, the request id, every context value set during the request, inline +messages, the AI usage rollup and any coded error. Summaries are **always kept**: +there is no sampler. The only volume controls are `ignore`, field budgets and +the ingester's `LOG_TTL_DAYS`. + +```ts +import express from "express"; +import { initAutterServer, autterRequests, autterErrorResponse } from "@autter/runtime-node"; + +initAutterServer({ apiKey: process.env.AUTTER_RUNTIME_KEY!, service: "payments-api" }); + +const app = express(); +app.use(autterRequests({ ignore: ["/healthz", "/metrics"] })); // early, before routers +app.use(express.json()); +// … routes … +app.use(autterErrorResponse()); // last +``` + +| Framework | Entry point | +| --- | --- | +| Express / Connect | `app.use(autterRequests(options))` | +| Fastify 4/5 | `app.register(autterFastify, options)` (no fastify dependency; skips encapsulation) | +| fetch-style handlers (Hono on Node, Remix, plain `Request → Response`) | `withRuntimeRequest(handler, { name?, route?, waitUntil? })` | +| Next.js route handlers | `withRuntimeRequest` from `@autter/runtime-next` (wires `after()`) | +| Plain `node:http` | `logging: { requests: true }` — experimental, see below | +| Workers / Vercel Edge / Deno / Bun | `withAutter` from `@autter/runtime-edge` | + +Options: `ignore` (path globs; `*` is one segment, `**` any depth), +`requestIdHeader` (default `x-request-id`), `trustRequestId` (default `true`). + +**Name and route.** The summary is named `METHOD /route/template`. Express and +Fastify supply the matched template (`/api/users/:id`). Fetch wrappers use +`name`/`route` when given, otherwise the pathname with id-like segments +(numbers, UUIDs, long hex) replaced by `:id` — the ingester's rules. + +**Attributes.** `autter.operation.kind = "request"`, `http.request.method`, +`http.route`, `http.response.status_code`, `autter.request.aborted` +(`close` before `finish`), `autter.request.id`, plus everything below. + +**Outcome.** An explicit `runtimeContext.outcome()` wins. Otherwise an +**expected** coded error → `degraded`; a thrown error (one that reached the +error middleware or escaped a fetch handler) or status ≥ 500 → `failed`; an +aborted request → `cancelled`; anything else → `succeeded`. + +### Request ids + +An inbound `x-request-id` matching `^[\w.-]{8,128}$` is honoured; anything else +is replaced with `crypto.randomUUID()`. The id is set on the response header, on +the server span (`autter.request.id`), and on every record inside the request, +including child operations. When the response carries CORS headers, the header +name is appended to `Access-Control-Expose-Headers` so a browser on another +origin can read it — runtime-browser 1.4.0 records it on failed fetch/XHR +calls, linking browser errors to the server request. + +Give the id to users: `runtimeContext.requestId` in emails or support pages, and +`autterErrorResponse` puts it in every error body. Autter's "Find request" +looks it up. + +## `runtimeContext` + +The current request or operation, from anywhere in its async call tree. Outside +one, `set`/`outcome` are no-ops and the log methods behave like `runtimeLogger`. + +```ts +import { runtimeContext } from "@autter/runtime-node"; + +runtimeContext.set({ cart: { items: 3 }, plan: "pro" }); // deep-merged, redacted +runtimeContext.info("Applied coupon", { coupon: "SPRING" }); +runtimeContext.warn("Price service slow", { ms: 900 }); +runtimeContext.error(err); // logs + attaches code/why/fix +runtimeContext.outcome("degraded", "Used cached prices"); +runtimeContext.requestId; // also .id (operation id) +``` + +### Inline messages + +Inside an operation, `debug`/`info` are folded into the summary as +`autter.operation.logs` (`{ t, level, message, attrs? }`, `t` = ms since start) +instead of becoming separate records. `warn`/`error` are folded **and** emitted. +`autter.operation.level` is the highest level seen. Limits: 50 messages +(then `autter.operation.logs_truncated`), 300-character messages, ~6 KB per +timeline. `logging.minLevel` still applies. Opt out with `logging.inline: false`. + +## Coded errors + +```ts +import { defineRuntimeErrors, RuntimeError } from "@autter/runtime-node"; // or runtime-edge + +export const billingErrors = defineRuntimeErrors("billing", { + declined: { + status: 402, message: "Payment declined", expected: true, + why: "The card issuer rejected the charge", + fix: "Ask the customer for another card", + link: "https://docs.example.com/payments#declined", + }, + limit: ({ plan }: { plan: string }) => ({ + status: 429, message: `Plan ${plan} limit reached`, fix: "Upgrade the plan", + }), +}); + +throw billingErrors.limit({ plan: "free" }); // code "billing.limit" +throw billingErrors.declined({ cause: stripeError, internal: { chargeId } }); +throw new RuntimeError({ code: "inventory.reservation_timeout", message, why, fix, cause }); +``` + +A code groups every occurrence into **one issue** (fingerprint scheme +`code-v1`: service + code), across message variants, routes and sources +(SDK traces, promoted logs, Sentry/PostHog/Datadog/Loki records carrying the +same code). Route and message remain facets of the issue. + +**Code rules** — `^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$`, at most 80 characters: + +- namespaced (`billing.declined`, not `declined`), stable across releases; +- low-cardinality: never ids, emails, amounts or anything from user input; +- an invalid code is dropped with a one-time console warning and the error + groups by message, exactly as before. + +Fields: `message` (client-safe), `status`, `expected`, `why` (≤ 1000 chars), +`fix` (≤ 1000), `link` (≤ 500). Per throw: `cause` (chains become +`exception.cause.N.type|message|code`, N ≤ 5), `internal` (debug payload, +redacted and attached to the **span only** as `autter.error.internal`; never in +logs, summaries or client responses; not enumerable, so `JSON.stringify` and +loggers skip it), and `message` to override the catalog text. + +**Existing error classes need no rewrite.** `captureException` duck-types +`code`/`why`/`fix`/`link`/`status` (or `statusCode`)/`expected` on any error: + +```ts +class AppError extends Error { + code = "auth.session_expired"; status = 401; expected = true; +} +``` + +Foreign codes that don't match the pattern (`ECONNREFUSED`, `42P01`) are ignored +silently. `isRuntimeErrorLike(err)` tells you whether an error has any of these +fields. + +**Expected errors** (`expected: true`) are business failures — declines, +validation, quota. They are recorded (occurrence with `autter.error.expected`, +summary outcome `degraded`) but never open incidents or trigger auto-fix; Autter +alerts only when their rate jumps. `why`/`fix` are shown in Autter as +"declared by the application" and treated as hypotheses by root-cause analysis, +never as proof. + +### `autterErrorResponse` and `toClientError` + +```ts +app.use(autterErrorResponse()); // Express error middleware, after routes +// 402 { "error": { "message": "Payment declined", "code": "billing.declined", +// "why": "…", "fix": "…", "requestId": "…" } } +``` + +- Status: the error's own 4xx/5xx `status`, else 500. +- Body: `toClientError(err, requestId)` → `{ error: { message, code?, why?, + fix?, link?, requestId? } }`. Never `internal`, stacks or causes. The message + of an unstructured 5xx error (`throw new Error("SELECT … failed")`) is + replaced with `"Internal Server Error"`. +- Reporting (`capture`, default): errors with status ≥ 500, RuntimeErrors and + any error with a valid code are reported via `captureException` (once, even + if the handler already captured it). Pass `capture: false` or a predicate. +- The error is recorded on the request summary either way. + +Use `toClientError` directly in Fastify `setErrorHandler`, Next.js route +handlers or anywhere else you build responses. runtime-browser's +`autterErrorFromResponse(res)` turns such a body back into an `Error` with the +same `code`/`why`/`fix`. + +## Background work: `fork`, `runInBackground`, carriers + +```ts +// Child linked to the current request even if it finishes later. +await runtimeContext.fork("send-receipt", async (op) => sendReceipt(order)); + +// Not awaited; errors are recorded on the child, never unhandled rejections. +runInBackground("warm-cache", () => warmCache(user)); + +// Across a queue or process: +await queue.add("charge", { orderId, autter: runtimeContext.carrier() }); +// consumer +await withRuntimeOperation("charge", (op) => charge(job.data), {}, { from: job.data.autter }); +``` + +`fork` captures the parent operation **at call time**, so the link survives +after the parent has sealed (plain `withRuntimeOperation` keeps 1.4.0 +behaviour: no link once the parent finished). `carrier()` returns +`{ v: 1, op, req?, traceparent? }` (JSON-safe, ~200 bytes). The consumer's +summary gets `autter.operation.parent_id = op`, the same `autter.request.id`, and +`autter.parent_trace_id`; its span starts in a new trace with an OTel **link** +to the producer span. Malformed carriers are ignored. + +## AI usage rollup + +`withLlmCall`, `trackLlmCall` and `instrumentLlmClient` also add each call to +the current operation's `autter.operation.ai`: +`{ calls, input_tokens, output_tokens, cache_read_tokens, cost_usd, models[] }`. +`cost_usd` sums the costs you report (`setCost`, `costUsd`); per-call spans still +get the ingester's price-table estimate. Cache reads come from +`setUsage({ cacheReadTokens })` or the provider response (OpenAI +`prompt_tokens_details.cached_tokens`, Anthropic `cache_read_input_tokens`). +The rollup is per operation (not added to ancestors), so totals never double count. + +## `waitUntil` (serverless) + +```ts +export const handler = withRuntimeRequest(fn, { waitUntil: (p) => ctx.waitUntil(p) }); +await withRuntimeOperation("job", fn, {}, { waitUntil }); +``` + +After the summary is emitted, the log flush promise is handed to `waitUntil` +instead of being left to the 2-second timer, so platforms that freeze after +the response (Vercel, Workers, Lambda response streaming) still deliver it. +`@autter/runtime-next`'s `withRuntimeRequest` wires Next's `after()` +automatically (Next 15+, `unstable_after` on 14.2; fails soft elsewhere). + +## Logger-only mode: `initAutterLogging` + +For apps that already run their own OpenTelemetry SDK (a second NodeSDK would +conflict) or want logs without tracing: + +```ts +import { initAutterLogging } from "@autter/runtime-node"; + +initAutterLogging({ + apiKey: process.env.AUTTER_RUNTIME_KEY, // omit → console/file only + service: "api", release: process.env.GIT_SHA, + logging: { enrich: [enrichDeployment()] }, +}); +``` + +Requests, operations, `runtimeContext`, coded errors, sinks and enrichers all +work; NodeSDK is never started (only `@opentelemetry/api` is read). Exceptions +(`captureException`, errors thrown out of operations, uncaught crashes) are +recorded on the app's active recording span when there is one; otherwise they +become error records with `exception.*`, `autter.error.*` and +`autter.capture.mode = "log"`, which ingester 1.5.0 promotes to occurrences +(deduplicated by trace id). Use `exceptions: "log"` when your own spans are not +exported to Autter. Do not call both `initAutterServer` and `initAutterLogging`. + +## Enrichers + +`logging.enrich` runs on every operation/request summary **before** redaction +and bounding: + +```ts +logging: { enrich: [enrichUserAgent(), enrichRequestSize(), enrichEdgeGeo(), enrichDeployment(), + (event, { req, operation }) => { event.attributes["tenant.tier"] = tierOf(req); }] } +``` + +| Enricher | Adds | +| --- | --- | +| `enrichUserAgent()` | `user_agent.browser` ("Chrome 120"), `user_agent.os`, `user_agent.device` (desktop/mobile/tablet/bot), `user_agent.bot` — same mapping as runtime-browser; never the raw string | +| `enrichRequestSize()` | `http.request.body.size`, `http.response.body.size` from Content-Length | +| `enrichEdgeGeo()` | `geo.country.iso_code` from Cloudflare/Vercel/CloudFront/Fastly headers — country only | +| `enrichDeployment()` | `deployment.region`, `deployment.commit` from common platform env vars | + +## Sinks and local files + +`logging.sinks` replaces the default `[otlpSink(), consoleSink()]`: + +- `otlpSink({ endpoint?, apiKey? })` — `/v1/logs` with the 1.4.0 buffering and retry limits. +- `consoleSink({ format?: "pretty" | "json" })` — `json` (1.4.0 lines) when + `NODE_ENV=production`, otherwise a one-line summary plus an indented tree of + context, steps, error and inline messages. ANSI colour only on a TTY + (`NO_COLOR` respected). +- `fileSink({ dir = ".autter/runtime", maxFiles = 7, maxBytes = 10 MiB })` — + NDJSON, one record per line, files `YYYY-MM-DD.jsonl` (UTC) then + `YYYY-MM-DD.N.jsonl` past `maxBytes`, oldest deleted beyond `maxFiles`. On by + default **only when `NODE_ENV=development`** (or `logging.file: true | {…}`); + disables itself with one warning on read-only or permission-denied + filesystems. Add `.autter/` to `.gitignore`. Coding agents read these files + (or `autter logs --local`) to answer "why did that request fail?". + +Records are redacted and bounded before any sink sees them. + +## Testing — `@autter/runtime-node/testing` + +```ts +import { captureRuntime, expectOperation } from "@autter/runtime-node/testing"; + +const runtime = captureRuntime(); // in-memory; no ingester, no network +await request(app).post("/checkout?declined").expect(402); +expectOperation(runtime, "POST /checkout") + .toHaveKind("request") + .toHaveOutcome("degraded") + .toHaveErrorCode("billing.declined") + .toHaveContext({ cart: { items: 3 } }) + .toHaveLog("Validating cart"); +runtime.byRequestId(id); runtime.exceptions; runtime.clear(); runtime.stop(); +``` + +`memorySink()` is also exported for custom sink lists. Capture works with or +without `initAutterServer`; before init it replaces console output. + +## Zero-code hook mode (experimental) + +`initAutterServer({ logging: { requests: true } })` creates request summaries for +every incoming `node:http` request from the HTTP instrumentation hook — no +middleware. It enters the request context with `AsyncLocalStorage.enterWith`, +which can leak into later work on the same socket under keep-alive or +pipelining, so it is **off by default** until a leak test proves it safe. Prefer +`autterRequests()`; when both are present the middleware reuses the hook's +summary. Express route templates are still captured. + +## Edge: `@autter/runtime-edge` + +```ts +import { withAutter, defineRuntimeErrors } from "@autter/runtime-edge"; + +export default withAutter( + (env) => ({ apiKey: env.AUTTER_RUNTIME_KEY, service: "edge-api" }), + async (request, env, ctx, rt) => { + rt.set({ tenant: env.TENANT }); + rt.info("Routing"); + if (!ok) throw billingErrors.declined(); // same catalogs as Node + return new Response("ok"); + }, +); +``` + +Zero dependencies, fetch-only, no AsyncLocalStorage — `rt` is the request +context (`set`, `outcome`, `info`, `warn`, `error`, `requestId`, +`captureException`). Same request summary, request-id and coded-error semantics +as Node; exceptions are promoted log records. Delivery uses `ctx.waitUntil` +(Workers) or `event.waitUntil` (Next middleware) and needs a **server** key — +never ship it to a browser. Next.js `middleware.ts` imports it as +`@autter/runtime-next/edge`. See [packages/runtime-edge](../packages/runtime-edge/README.md). + +## Wire reference + +| Attribute | Where | +| --- | --- | +| `autter.request.id` | request summaries, every record inside a request, server span | +| `autter.operation.kind` | every summary (`request` or `operation`) | +| `autter.operation.logs`, `.logs_truncated`, `.level` | summaries | +| `autter.operation.ai` | summaries with LLM calls | +| `http.request.method`, `http.route`, `http.response.status_code`, `autter.request.aborted` | request summaries | +| `autter.error.code` / `.why` / `.fix` / `.link` / `.status` / `.expected` | span, exception event, summaries, error records | +| `autter.error.internal` | span only (redacted JSON) | +| `exception.cause.N.type` / `.message` / `.code` | exception event, summaries, error records | +| `autter.capture.mode = "log"` | promoted error records (logger-only, edge) | +| `autter.parent_trace_id` | consumer operations started from a carrier | diff --git a/examples/edge-worker/README.md b/examples/edge-worker/README.md new file mode 100644 index 0000000..b7ba5c6 --- /dev/null +++ b/examples/edge-worker/README.md @@ -0,0 +1,16 @@ +# Edge worker example + +A Cloudflare Worker using [`@autter/runtime-edge`](../../packages/runtime-edge): +request summaries, request ids, and coded errors with zero dependencies. + +```bash +npx wrangler secret put AUTTER_RUNTIME_KEY # a SERVER ingest key, never a browser key +npx wrangler dev +curl -i localhost:8787/?plan=pro # 200, x-request-id echoed +curl -i "localhost:8787/?over" # 429 { error: { code: "quota.exceeded", … } } +``` + +`wrangler.toml` is a sketch; this repo does not depend on Wrangler. The same +handler works on Vercel Edge (`export default`), Deno (`Deno.serve(handler)`) +and Bun (`Bun.serve(handler)`), where `waitUntil` is not available and +delivery is fire-and-forget. diff --git a/examples/edge-worker/package.json b/examples/edge-worker/package.json new file mode 100644 index 0000000..7007ce7 --- /dev/null +++ b/examples/edge-worker/package.json @@ -0,0 +1,14 @@ +{ + "name": "example-edge-worker", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "build": "node --check src/index.js", + "dev": "wrangler dev", + "deploy": "wrangler deploy" + }, + "dependencies": { + "@autter/runtime-edge": "^1.0.0" + } +} diff --git a/examples/edge-worker/src/index.js b/examples/edge-worker/src/index.js new file mode 100644 index 0000000..c2bde10 --- /dev/null +++ b/examples/edge-worker/src/index.js @@ -0,0 +1,40 @@ +/** + * Example: Cloudflare Worker instrumented with @autter/runtime-edge. + * + * One request summary per request (method, route, status, outcome, + * duration, request id, context, inline messages), coded errors as + * occurrences, delivery through ctx.waitUntil. No AsyncLocalStorage: the + * request context is the `rt` argument. + */ +import { withAutter, defineRuntimeErrors } from "@autter/runtime-edge"; + +const quotaErrors = defineRuntimeErrors("quota", { + exceeded: ({ plan }) => ({ + status: 429, + message: `Plan ${plan} request quota exceeded`, + expected: true, + fix: "Upgrade the plan or retry after the window resets", + }), +}); + +export default withAutter( + // Bindings only exist per request, so options can be a function of env. + (env) => ({ + apiKey: env.AUTTER_RUNTIME_KEY, + endpoint: env.AUTTER_ENDPOINT, + service: "edge-example", + environment: env.ENVIRONMENT, + ignore: ["/healthz"], + // Thrown errors become `{ error: { message, code, fix, requestId } }`. + errorResponse: true, + }), + async (request, env, _ctx, rt) => { + const url = new URL(request.url); + if (url.pathname === "/healthz") return new Response("ok"); + const plan = url.searchParams.get("plan") ?? "free"; + rt.set({ plan, colo: request.cf?.colo }); + rt.info("Checking quota"); + if (url.searchParams.has("over")) throw quotaErrors.exceeded({ plan }); + return Response.json({ ok: true, requestId: rt.requestId }); + }, +); diff --git a/examples/edge-worker/wrangler.toml b/examples/edge-worker/wrangler.toml new file mode 100644 index 0000000..743a442 --- /dev/null +++ b/examples/edge-worker/wrangler.toml @@ -0,0 +1,9 @@ +# Sketch only — `wrangler` is not a dependency of this repo. +# npx wrangler secret put AUTTER_RUNTIME_KEY (a SERVER ingest key) +name = "autter-edge-example" +main = "src/index.js" +compatibility_date = "2026-09-01" + +[vars] +AUTTER_ENDPOINT = "https://otlp.autter.dev" +ENVIRONMENT = "production" diff --git a/examples/express-app/package.json b/examples/express-app/package.json index de6be65..16a6dea 100644 --- a/examples/express-app/package.json +++ b/examples/express-app/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@autter/runtime-browser": "^1.0.0", - "@autter/runtime-node": "^1.0.0", + "@autter/runtime-node": "^1.5.0", "express": "^4.21.2" } } diff --git a/examples/express-app/server.js b/examples/express-app/server.js index 8aa6352..e587e7a 100644 --- a/examples/express-app/server.js +++ b/examples/express-app/server.js @@ -4,6 +4,8 @@ * AUTTER_RUNTIME_KEY=dev-key AUTTER_ENDPOINT=http://localhost:4318 node server.js * * - Server tracing/errors via @autter/runtime-node (OTel → /v1/traces) + * - One request summary per HTTP request via autterRequests (→ /v1/logs), + * echoed request ids, coded errors and client-safe error responses * - Browser errors via @autter/runtime-browser → same-origin relay → /v1/browser */ import { @@ -12,6 +14,10 @@ import { captureException, withLlmCall, emitLlmSelftestTrace, + autterRequests, + autterErrorResponse, + defineRuntimeErrors, + runtimeContext, } from "@autter/runtime-node"; const endpoint = process.env.AUTTER_ENDPOINT ?? "https://otlp.autter.dev"; @@ -28,10 +34,31 @@ initAutterServer({ traceSampleRate: 1, // sample everything in the example }); +// Coded errors: one code = one issue, whatever the message says. `expected` +// failures are recorded but never open incidents. +const checkoutErrors = defineRuntimeErrors("checkout", { + card_declined: { + status: 402, + message: "Payment declined", + expected: true, + why: "The card issuer rejected the charge", + fix: "Ask the customer for another card", + }, + out_of_stock: ({ sku }) => ({ + status: 409, + message: `Item ${sku} is out of stock`, + fix: "Remove the item or wait for restock", + }), +}); + const { default: express } = await import("express"); const app = express(); +// One `kind: "request"` summary per request (always kept), with the request +// id echoed in `x-request-id`. Health checks are not worth a record. +app.use(autterRequests({ ignore: ["/healthz"] })); app.use(express.static("public")); +app.get("/healthz", (_req, res) => res.send("ok")); // Same-origin relay: the browser posts here; the key stays server-side. app.post("/api/autter-runtime", createBrowserRelayHandler({ apiKey, endpoint })); @@ -40,7 +67,21 @@ app.get("/api/ok", (_req, res) => res.json({ ok: true })); // Param route: shows up in runtime_metrics_1m as "/api/users/:id" — one // rollup row per route template, not one per user id. -app.get("/api/users/:id", (req, res) => res.json({ id: req.params.id })); +app.get("/api/users/:id", (req, res) => { + runtimeContext.set({ user: { id: req.params.id } }); + res.json({ id: req.params.id, requestId: runtimeContext.requestId }); +}); + +// Coded errors flow to autterErrorResponse below: +// 402 { error: { message, code: "checkout.card_declined", why, fix, requestId } } +app.post("/api/checkout", express.json(), (req, res, next) => { + runtimeContext.set({ cart: { items: req.body?.items ?? 0 } }); + runtimeContext.info("Validating cart"); + if (req.query.declined !== undefined) return next(checkoutErrors.card_declined()); + if (req.query.sku) return next(checkoutErrors.out_of_stock({ sku: String(req.query.sku) })); + runtimeContext.outcome("succeeded"); + res.json({ ok: true, requestId: runtimeContext.requestId }); +}); app.get("/api/boom", (_req, res) => { try { @@ -71,6 +112,10 @@ app.get("/api/llm-selftest", async (_req, res) => { res.json(await emitLlmSelftestTrace()); }); +// Last: turns errors into client-safe JSON (never `internal`), records them on +// the request summary and reports coded/5xx errors. +app.use(autterErrorResponse()); + app.listen(3000, () => { console.log("example app on http://localhost:3000 (open it, then click the buttons)"); }); diff --git a/package-lock.json b/package-lock.json index 8d5c550..07b4f9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,12 +16,19 @@ "node": ">=20" } }, + "examples/edge-worker": { + "name": "example-edge-worker", + "version": "0.0.0", + "dependencies": { + "@autter/runtime-edge": "^1.0.0" + } + }, "examples/express-app": { "name": "example-express-app", "version": "0.0.0", "dependencies": { "@autter/runtime-browser": "^1.0.0", - "@autter/runtime-node": "^1.0.0", + "@autter/runtime-node": "^1.5.0", "express": "^4.21.2" } }, @@ -33,6 +40,14 @@ "resolved": "packages/runtime-browser", "link": true }, + "node_modules/@autter/runtime-core": { + "resolved": "packages/runtime-core", + "link": true + }, + "node_modules/@autter/runtime-edge": { + "resolved": "packages/runtime-edge", + "link": true + }, "node_modules/@autter/runtime-next": { "resolved": "packages/runtime-next", "link": true @@ -2717,6 +2732,10 @@ "node": ">= 0.6" } }, + "node_modules/example-edge-worker": { + "resolved": "examples/edge-worker", + "link": true + }, "node_modules/example-express-app": { "resolved": "examples/express-app", "link": true @@ -4667,7 +4686,7 @@ }, "packages/otlp-ingester": { "name": "@autter/otlp-ingester", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "@clickhouse/client": "^1.12.0", @@ -4778,7 +4797,7 @@ }, "packages/runtime-browser": { "name": "@autter/runtime-browser", - "version": "1.3.3", + "version": "1.4.0", "license": "MIT", "devDependencies": { "@size-limit/preset-small-lib": "^11.1.6", @@ -4787,13 +4806,36 @@ "typescript": "^5.7.0" } }, + "packages/runtime-core": { + "name": "@autter/runtime-core", + "version": "0.0.0", + "license": "MIT", + "devDependencies": { + "tsup": "^8.3.5", + "typescript": "^5.7.0" + } + }, + "packages/runtime-edge": { + "name": "@autter/runtime-edge", + "version": "1.0.0", + "license": "MIT", + "devDependencies": { + "@autter/runtime-core": "0.0.0", + "tsup": "^8.3.5", + "typescript": "^5.7.0" + }, + "engines": { + "node": ">=20" + } + }, "packages/runtime-next": { "name": "@autter/runtime-next", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@autter/runtime-browser": "^1.3.3", - "@autter/runtime-node": "^1.4.0" + "@autter/runtime-browser": "^1.4.0", + "@autter/runtime-edge": "^1.0.0", + "@autter/runtime-node": "^1.5.0" }, "devDependencies": { "@types/react": "^18.3.0", @@ -4809,7 +4851,7 @@ }, "packages/runtime-node": { "name": "@autter/runtime-node", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "@opentelemetry/api": "^1.9.0", @@ -4824,7 +4866,9 @@ "@opentelemetry/semantic-conventions": "^1.28.0" }, "devDependencies": { + "@autter/runtime-core": "0.0.0", "@types/node": "^22.10.0", + "express": "^4.21.2", "tsup": "^8.3.5", "typescript": "^5.7.0" }, diff --git a/package.json b/package.json index 75480a6..ff71da7 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,9 @@ "examples/*" ], "scripts": { - "build": "npm run build -w @autter/runtime-browser -w @autter/runtime-node -w @autter/otlp-ingester && npm run build -w @autter/runtime-next", - "dev": "npm run dev -w @autter/otlp-ingester" + "build": "npm run build -w @autter/runtime-core && npm run build -w @autter/runtime-node -w @autter/runtime-edge -w @autter/runtime-browser -w @autter/otlp-ingester && npm run build -w @autter/runtime-next", + "dev": "npm run dev -w @autter/otlp-ingester", + "test": "npm run test -w @autter/runtime-core -w @autter/runtime-node -w @autter/runtime-edge -w @autter/runtime-next" }, "engines": { "node": ">=20" diff --git a/packages/runtime-core/fixtures/code-fingerprints.json b/packages/runtime-core/fixtures/code-fingerprints.json new file mode 100644 index 0000000..d5aa010 --- /dev/null +++ b/packages/runtime-core/fixtures/code-fingerprints.json @@ -0,0 +1,6 @@ +[ + { "service": "payments-api", "code": "billing.declined", "fingerprint": "1692d304df5e4edbd3a0bbac5d7658bb" }, + { "service": "payments-api", "code": "billing.limit", "fingerprint": "be8302686d461915fb67bd28cdba26ba" }, + { "service": "web-app", "code": "auth.session_expired", "fingerprint": "b997fe46c7a61540917324f78b710d83" }, + { "service": "autter-api", "code": "github.app_permission_missing", "fingerprint": "ce80dc4c785f51a0851e99cd5dd1316f" } +] diff --git a/packages/runtime-core/package.json b/packages/runtime-core/package.json new file mode 100644 index 0000000..ea162bb --- /dev/null +++ b/packages/runtime-core/package.json @@ -0,0 +1,26 @@ +{ + "name": "@autter/runtime-core", + "private": true, + "version": "0.0.0", + "description": "Internal, unpublished code shared by @autter/runtime-node and @autter/runtime-edge (bundled into both at build time)", + "license": "MIT", + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "sideEffects": false, + "scripts": { + "build": "tsup src/index.ts --format esm --dts --target es2022 --clean", + "test": "npm run build && node --test test/*.test.mjs" + }, + "devDependencies": { + "tsup": "^8.3.5", + "typescript": "^5.7.0" + } +} diff --git a/packages/runtime-core/src/attributes.ts b/packages/runtime-core/src/attributes.ts new file mode 100644 index 0000000..afcf44b --- /dev/null +++ b/packages/runtime-core/src/attributes.ts @@ -0,0 +1,13 @@ +/** + * Structural copy of OpenTelemetry's `Attributes` so runtime-core stays + * dependency-free (it is bundled into the zero-dependency edge package). + * Assignable both ways with `@opentelemetry/api`'s type. + */ +export type AttributeValue = + | string + | number + | boolean + | Array + | Array + | Array; +export type Attributes = Record; diff --git a/packages/runtime-core/src/carrier.ts b/packages/runtime-core/src/carrier.ts new file mode 100644 index 0000000..5bd646e --- /dev/null +++ b/packages/runtime-core/src/carrier.ts @@ -0,0 +1,85 @@ +import { REQUEST_ID_PATTERN } from "./request.js"; + +/** + * Serialisable link from a producer operation to work it hands off (a queue + * job, another process): `{ v: 1, op, req?, traceparent? }`. Put it in the + * job payload; the consumer passes it as `withRuntimeOperation(..., { from })`. + */ +export interface RuntimeCarrier { + v: 1; + /** Producer operation id (becomes the consumer's `autter.operation.parent_id`). */ + op: string; + /** Request id the work belongs to. */ + req?: string; + /** W3C traceparent of the producer span (consumer links to it). */ + traceparent?: string; +} + +const OPERATION_ID = /^[\w.-]{1,128}$/; +const TRACEPARENT = /^00-([a-f0-9]{32})-([a-f0-9]{16})-([a-f0-9]{2})$/; + +export function formatTraceparent( + traceId: string, + spanId: string, + sampled: boolean, +): string | undefined { + const value = `00-${traceId}-${spanId}-${sampled ? "01" : "00"}`; + return TRACEPARENT.test(value) && !/^0+$/.test(traceId) && !/^0+$/.test(spanId) + ? value + : undefined; +} + +export function parseTraceparent( + value: unknown, +): { traceId: string; spanId: string; sampled: boolean } | null { + if (typeof value !== "string") return null; + const match = TRACEPARENT.exec(value.trim()); + if (!match || /^0+$/.test(match[1]!) || /^0+$/.test(match[2]!)) return null; + return { + traceId: match[1]!, + spanId: match[2]!, + sampled: (Number.parseInt(match[3]!, 16) & 1) === 1, + }; +} + +export function createCarrier(input: { + op: string; + req?: string; + traceparent?: string; +}): RuntimeCarrier { + return { + v: 1, + op: input.op, + ...(input.req ? { req: input.req } : {}), + ...(input.traceparent ? { traceparent: input.traceparent } : {}), + }; +} + +/** + * Validate a carrier from an untrusted payload (object or JSON string). + * Returns null for anything malformed; invalid optional fields are dropped. + */ +export function parseCarrier(input: unknown): RuntimeCarrier | null { + let value = input; + if (typeof value === "string") { + if (value.length > 1024) return null; + try { + value = JSON.parse(value); + } catch { + return null; + } + } + if (!value || typeof value !== "object") return null; + const raw = value as Record; + if (raw.v !== 1 || typeof raw.op !== "string" || !OPERATION_ID.test(raw.op)) + return null; + return createCarrier({ + op: raw.op, + ...(typeof raw.req === "string" && REQUEST_ID_PATTERN.test(raw.req) + ? { req: raw.req } + : {}), + ...(parseTraceparent(raw.traceparent) + ? { traceparent: String(raw.traceparent).trim() } + : {}), + }); +} diff --git a/packages/runtime-core/src/context.ts b/packages/runtime-core/src/context.ts new file mode 100644 index 0000000..526e2a7 --- /dev/null +++ b/packages/runtime-core/src/context.ts @@ -0,0 +1,162 @@ +import type { Attributes } from "./attributes.js"; +import { redactAttributes } from "./redact.js"; + +export type RuntimeLogContextValue = + | string + | number + | boolean + | null + | undefined + | RuntimeLogContextValue[] + | { [key: string]: RuntimeLogContextValue }; +export type RuntimeLogContext = Record; +export type RuntimeLogLevel = "debug" | "info" | "warning" | "error"; +export type RuntimeOutcome = + | "succeeded" + | "failed" + | "degraded" + | "cancelled" + | "pending"; +export type RuntimeOperationKind = "request" | "operation"; + +/** OTLP severity numbers for the four runtime levels. */ +export const LEVEL_SEVERITY: Record = { + debug: 5, + info: 9, + warning: 13, + error: 17, +}; + +/** Higher of two levels (by severity). */ +export function maxLevel( + a: RuntimeLogLevel, + b: RuntimeLogLevel, +): RuntimeLogLevel { + return LEVEL_SEVERITY[b] > LEVEL_SEVERITY[a] ? b : a; +} + +export type Redactor = (attributes: Attributes) => Attributes; + +/** Attribute families the SDK owns; user context can never set them. */ +const RESERVED_KEY = + /^autter\.(?:operation|event|request|error)\.|^autter\.(?:parent_trace_id|capture\.mode)$/; + +/** Depth-0 ordering: SDK-owned keys first, exception/error next, user keys, + * then the inline log timeline last so it can never starve user context of + * the shared node/character budget. Stable for keys 1.4.0 already emitted. */ +function priority(key: string): number { + if (key === "autter.operation.logs") return -1; + if (/^autter\.(?:operation|event|request)\.|^autter\.parent_trace_id$/.test(key)) + return 2; + if (/^(?:exception|autter\.error)\./.test(key)) return 1; + return 0; +} + +/** + * Redact, then bound a context tree: 512 nodes, depth 6, 16 KB of string + * characters, 64 array items, 100 keys per object, 200-character keys, URL + * query strings stripped. Sets `autter.context.truncated` when anything was + * cut. Byte-compatible with runtime-node 1.4.0's `safe()`. + */ +export function boundContext( + attributes: RuntimeLogContext = {}, + redact?: Redactor, +): RuntimeLogContext { + const redacted = redact + ? redact(attributes as Attributes) + : redactAttributes(attributes as Attributes); + let budget = 512; + let characters = 16384; + let truncated = false; + const visit = (input: unknown, depth: number): unknown => { + if (--budget < 0 || depth > 6) { + truncated = true; + return "[truncated]"; + } + if (typeof input === "string") { + const text = input.replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); + const kept = text.slice(0, Math.max(0, characters)); + truncated ||= kept.length < text.length; + characters -= kept.length; + return kept; + } + if (typeof input === "number") + return Number.isFinite(input) ? input : undefined; + if (typeof input === "boolean" || input === null) return input; + if (Array.isArray(input)) { + truncated ||= input.length > 64; + return input.slice(0, 64).map((item) => visit(item, depth + 1)); + } + if (input && typeof input === "object") { + const entries = Object.entries(input); + if (depth === 0) entries.sort(([a], [b]) => priority(b) - priority(a)); + truncated ||= entries.length > 100; + return Object.fromEntries( + entries + .slice(0, 100) + .filter(([key]) => !/^(?:__proto__|constructor|prototype)$/.test(key)) + .map(([key, item]) => [key.slice(0, 200), visit(item, depth + 1)]), + ); + } + return undefined; + }; + const result = visit(redacted, 0) as RuntimeLogContext; + if (truncated) result["autter.context.truncated"] = true; + return result; +} + +/** Bounded user context with SDK-owned keys removed (anti-spoofing). */ +export function userContext( + attributes: RuntimeLogContext = {}, + redact?: Redactor, +): RuntimeLogContext { + return Object.fromEntries( + Object.entries(boundContext(attributes, redact)).filter( + ([key]) => !RESERVED_KEY.test(key), + ), + ); +} + +/** Deep-merge `next` into `current` (objects merge to depth 6, everything + * else is replaced), then re-bound. */ +export function mergeContext( + current: RuntimeLogContext, + next: RuntimeLogContext = {}, + redact?: Redactor, +): RuntimeLogContext { + const merge = ( + left: RuntimeLogContext, + right: RuntimeLogContext, + depth: number, + ): RuntimeLogContext => { + const result = { ...left }; + for (const [key, value] of Object.entries(right)) { + const prior = result[key]; + result[key] = + depth < 6 && + value && + typeof value === "object" && + !Array.isArray(value) && + prior && + typeof prior === "object" && + !Array.isArray(prior) + ? merge(prior, value, depth + 1) + : value; + } + return result; + }; + return userContext(merge(current, userContext(next, redact), 0), redact); +} + +/** Flatten a context for span attributes: nested values become JSON strings, + * null/undefined are dropped. */ +export function toSpanAttributes(context: RuntimeLogContext): Attributes { + return Object.fromEntries( + Object.entries(context) + .filter(([, item]) => item !== null && item !== undefined) + .map(([key, item]) => [ + key, + typeof item === "object" ? JSON.stringify(item) : item, + ]), + ) as Attributes; +} diff --git a/packages/runtime-core/src/errors.ts b/packages/runtime-core/src/errors.ts new file mode 100644 index 0000000..0680a95 --- /dev/null +++ b/packages/runtime-core/src/errors.ts @@ -0,0 +1,358 @@ +/** + * Structured, coded errors shared by every Autter Runtime server SDK. + * + * export const billingErrors = defineRuntimeErrors("billing", { + * declined: { status: 402, message: "Payment declined", expected: true, + * why: "The card issuer rejected the charge", + * fix: "Ask the customer for another card" }, + * limit: ({ plan }: { plan: string }) => ({ status: 429, message: `Plan ${plan} limit reached` }), + * }); + * throw billingErrors.limit({ plan: "free" }); // code "billing.limit" + * + * A code groups every occurrence into one issue (scheme "code-v1"), so codes + * must be stable, namespaced and low-cardinality: never ids, user data or + * secrets. Anything not matching CODE_PATTERN is dropped and the error + * groups by message exactly as before. + */ + +/** `namespace.key` with up to four segments, lower-case, max 80 characters. */ +export const CODE_PATTERN = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$/; +export const CODE_MAX_LENGTH = 80; +const WHY_MAX = 1000; +const FIX_MAX = 1000; +const LINK_MAX = 500; +const CAUSE_DEPTH = 5; + +export function isValidErrorCode(code: unknown): code is string { + return ( + typeof code === "string" && + code.length <= CODE_MAX_LENGTH && + CODE_PATTERN.test(code) + ); +} + +const warned = new Set(); +/** One console warning per distinct invalid code (capped, never throws). */ +function warnInvalidCode(code: unknown): void { + const key = String(code).slice(0, 120); + if (warned.has(key) || warned.size >= 50) return; + warned.add(key); + try { + console.warn( + `[autter-runtime] error code ${JSON.stringify(key)} does not match ${CODE_PATTERN} (max ${CODE_MAX_LENGTH} chars); it is dropped and the error groups by message`, + ); + } catch { + /* console unavailable */ + } +} + +export interface RuntimeErrorDefinition { + /** Human-readable message. Safe to show to clients. */ + message: string; + /** HTTP-ish status. Default 500 when the error reaches a response. */ + status?: number; + /** Expected business failure (decline, validation): recorded, never an incident. */ + expected?: boolean; + /** Declared cause, ≤ 1000 chars. Treated as a hypothesis by Autter's RCA. */ + why?: string; + /** Declared remedy, ≤ 1000 chars. */ + fix?: string; + /** Docs link, ≤ 500 chars. */ + link?: string; +} + +export interface RuntimeErrorExtras { + /** Underlying error; becomes `exception.cause.N.*` (max 5 levels). */ + cause?: unknown; + /** Debug-only details: redacted and attached to the span only, never to logs or client responses. */ + internal?: Record; + /** Override the catalog message for this throw. */ + message?: string; +} + +export interface RuntimeErrorOptions extends RuntimeErrorDefinition { + /** Stable namespaced code, e.g. "inventory.reservation_timeout". */ + code?: string; + /** Underlying error; becomes `exception.cause.N.*` (max 5 levels). */ + cause?: unknown; + /** Debug-only details: redacted and attached to the span only, never to logs or client responses. */ + internal?: Record; +} + +/** The fields Autter reads from any error (duck-typed). */ +export interface RuntimeErrorLike { + message?: string; + code?: string; + why?: string; + fix?: string; + link?: string; + status?: number; + expected?: boolean; +} + +export class RuntimeError extends Error implements RuntimeErrorLike { + readonly code?: string; + readonly why?: string; + readonly fix?: string; + readonly link?: string; + readonly status?: number; + readonly expected?: boolean; + /** Non-enumerable so JSON.stringify / loggers never serialise it. */ + declare readonly internal?: Record; + + constructor(options: RuntimeErrorOptions | string) { + const opts: RuntimeErrorOptions = + typeof options === "string" ? { message: options } : options; + super( + opts.message, + opts.cause !== undefined ? { cause: opts.cause } : undefined, + ); + this.name = "RuntimeError"; + if (opts.code !== undefined) { + if (isValidErrorCode(opts.code)) this.code = opts.code; + else warnInvalidCode(opts.code); + } + if (typeof opts.why === "string") this.why = opts.why.slice(0, WHY_MAX); + if (typeof opts.fix === "string") this.fix = opts.fix.slice(0, FIX_MAX); + if (typeof opts.link === "string") this.link = opts.link.slice(0, LINK_MAX); + if (typeof opts.status === "number" && Number.isFinite(opts.status)) + this.status = Math.trunc(opts.status); + if (typeof opts.expected === "boolean") this.expected = opts.expected; + if (opts.internal !== undefined) + Object.defineProperty(this, "internal", { + value: opts.internal, + enumerable: false, + writable: false, + configurable: true, + }); + } + + /** Client-safe JSON (what `toClientError` returns, minus the wrapper). */ + toJSON(): Record { + return toClientError(this).error; + } +} + +type CatalogEntry = + | RuntimeErrorDefinition + // biome-ignore lint/suspicious/noExplicitAny: catalog factories take arbitrary argument shapes + | ((args: any) => RuntimeErrorDefinition); + +/** A typed factory for one catalog entry; `.code` is the namespaced code. */ +export type RuntimeErrorFactory = E extends ( + args: infer A, +) => RuntimeErrorDefinition + ? ((args: A, extras?: RuntimeErrorExtras) => RuntimeError) & { + readonly code: string; + } + : ((extras?: RuntimeErrorExtras) => RuntimeError) & { + readonly code: string; + }; + +export type RuntimeErrorCatalog> = { + readonly [K in keyof C]: RuntimeErrorFactory; +}; + +/** + * Build typed error factories from a catalog. Each key becomes the code + * `${namespace}.${key}`; an invalid resulting code is dropped with a + * one-time warning (the factory still produces a normal RuntimeError). + */ +export function defineRuntimeErrors>( + namespace: string, + catalog: C, +): RuntimeErrorCatalog { + const out: Record = {}; + for (const [key, entry] of Object.entries(catalog)) { + const code = `${namespace}.${key}`; + const valid = isValidErrorCode(code); + if (!valid) warnInvalidCode(code); + const build = (definition: RuntimeErrorDefinition, extras?: RuntimeErrorExtras) => { + const error = new RuntimeError({ + ...definition, + ...(extras?.message !== undefined ? { message: extras.message } : {}), + ...(valid ? { code } : {}), + ...(extras?.cause !== undefined ? { cause: extras.cause } : {}), + ...(extras?.internal !== undefined ? { internal: extras.internal } : {}), + }); + // Point the stack at the throw site, not this factory. + const captureStackTrace = ( + Error as unknown as { + captureStackTrace?: (target: object, ctor?: unknown) => void; + } + ).captureStackTrace; + captureStackTrace?.(error, factory); + return error; + }; + const factory = + typeof entry === "function" + ? (args: unknown, extras?: RuntimeErrorExtras) => + build((entry as (a: unknown) => RuntimeErrorDefinition)(args), extras) + : (extras?: RuntimeErrorExtras) => + build(entry as RuntimeErrorDefinition, extras); + Object.defineProperty(factory, "code", { value: code, enumerable: true }); + out[key] = factory; + } + return out as RuntimeErrorCatalog; +} + +/** + * True for any error carrying Autter's structured fields — a RuntimeError, + * or ANY object with a string `code`/`why`/`fix`/`link`, a numeric `status` + * (or `statusCode`) or a boolean `expected`. Existing error classes benefit + * without being rewritten. + */ +export function isRuntimeErrorLike(err: unknown): err is RuntimeErrorLike { + if (err instanceof RuntimeError) return true; + if (!err || typeof err !== "object") return false; + const e = err as Record; + return ( + typeof e.code === "string" || + typeof e.why === "string" || + typeof e.fix === "string" || + typeof e.link === "string" || + typeof e.status === "number" || + typeof e.statusCode === "number" || + typeof e.expected === "boolean" + ); +} + +function field(err: unknown, key: string): unknown { + if (!err || typeof err !== "object") return undefined; + try { + return (err as Record)[key]; + } catch { + return undefined; + } +} + +/** The valid code of an error, or undefined. Foreign codes (ECONNREFUSED, + * 42P01, …) are silently ignored; RuntimeErrors already warned at construction. */ +export function errorCodeOf(err: unknown): string | undefined { + const code = field(err, "code"); + return isValidErrorCode(code) ? code : undefined; +} + +/** HTTP-ish status of an error (`status`, then `statusCode`), or undefined. */ +export function errorStatusOf(err: unknown): number | undefined { + for (const key of ["status", "statusCode"]) { + const status = field(err, key); + if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) + return status; + } + return undefined; +} + +export function isExpectedError(err: unknown): boolean { + return field(err, "expected") === true; +} + +export interface ErrorAttributeOptions { + /** Also emit `exception.cause.N.*` for the cause chain (default true). */ + causes?: boolean; +} + +/** + * Wire attributes for an error: `autter.error.code|why|fix|link|status|expected` + * and `exception.cause.N.type|message|code` (N = 1..5). Never includes + * `internal` — see `errorInternal`. + */ +export function errorAttributes( + err: unknown, + options: ErrorAttributeOptions = {}, +): Record { + const out: Record = {}; + if (!err || typeof err !== "object") return out; + const code = errorCodeOf(err); + if (code) out["autter.error.code"] = code; + const why = field(err, "why"); + if (typeof why === "string" && why) out["autter.error.why"] = why.slice(0, WHY_MAX); + const fix = field(err, "fix"); + if (typeof fix === "string" && fix) out["autter.error.fix"] = fix.slice(0, FIX_MAX); + const link = field(err, "link"); + if (typeof link === "string" && link) out["autter.error.link"] = link.slice(0, LINK_MAX); + const status = errorStatusOf(err); + if (status !== undefined && isRuntimeErrorLike(err)) out["autter.error.status"] = status; + const expected = field(err, "expected"); + if (typeof expected === "boolean") out["autter.error.expected"] = expected; + if (options.causes !== false) { + let cause = field(err, "cause"); + const seen = new Set([err]); + for (let n = 1; n <= CAUSE_DEPTH && cause !== undefined && cause !== null; n++) { + if (seen.has(cause)) break; + seen.add(cause); + const isError = typeof cause === "object"; + const type = isError + ? String(field(cause, "name") ?? (cause as object).constructor?.name ?? "Error") + : typeof cause; + const message = isError ? field(cause, "message") : cause; + out[`exception.cause.${n}.type`] = type.slice(0, 200); + out[`exception.cause.${n}.message`] = String(message ?? "").slice(0, 1000); + const causeCode = field(cause, "code"); + if (typeof causeCode === "string" && causeCode) + out[`exception.cause.${n}.code`] = causeCode.slice(0, CODE_MAX_LENGTH); + cause = isError ? field(cause, "cause") : undefined; + } + } + return out; +} + +/** The `internal` debug payload of an error (span-only), if any. */ +export function errorInternal(err: unknown): Record | undefined { + const internal = field(err, "internal"); + return internal && typeof internal === "object" && !Array.isArray(internal) + ? (internal as Record) + : undefined; +} + +export interface ClientErrorBody { + error: { + message: string; + code?: string; + why?: string; + fix?: string; + link?: string; + requestId?: string; + }; +} + +const GENERIC_MESSAGE = "Internal Server Error"; + +/** + * Client-safe JSON body for an error: + * `{ error: { message, code?, why?, fix?, link?, requestId? } }`. + * Never includes `internal`, stacks or causes. The message of an + * unstructured 5xx error (a plain `throw new Error(...)`) is replaced with a + * generic one, because it was never written for clients. + */ +export function toClientError(err: unknown, requestId?: string): ClientErrorBody { + const code = errorCodeOf(err); + const status = errorStatusOf(err); + const declared = + err instanceof RuntimeError || + code !== undefined || + (status !== undefined && status < 500 && isRuntimeErrorLike(err)); + const rawMessage = field(err, "message"); + const message = + declared && typeof rawMessage === "string" && rawMessage + ? rawMessage.slice(0, 1000) + : GENERIC_MESSAGE; + const body: ClientErrorBody["error"] = { message }; + if (code) body.code = code; + if (declared) { + const why = field(err, "why"); + if (typeof why === "string" && why) body.why = why.slice(0, WHY_MAX); + const fix = field(err, "fix"); + if (typeof fix === "string" && fix) body.fix = fix.slice(0, FIX_MAX); + const link = field(err, "link"); + if (typeof link === "string" && link) body.link = link.slice(0, LINK_MAX); + } + if (typeof requestId === "string" && requestId) body.requestId = requestId; + return { error: body }; +} + +/** Status to answer with: the error's own 4xx/5xx status, else 500. */ +export function responseStatusOf(err: unknown): number { + const status = errorStatusOf(err); + return status !== undefined && status >= 400 ? status : 500; +} diff --git a/packages/runtime-core/src/fingerprint-code.ts b/packages/runtime-core/src/fingerprint-code.ts new file mode 100644 index 0000000..09c3b54 --- /dev/null +++ b/packages/runtime-core/src/fingerprint-code.ts @@ -0,0 +1,27 @@ +import { isValidErrorCode } from "./errors.js"; + +/** Fingerprint scheme for coded errors (see fixtures/code-fingerprints.json). */ +export const CODE_FINGERPRINT_SCHEME = "code-v1"; + +/** + * `sha256("code-v1" \0 service \0 code).hex().slice(0, 32)` — source + * independent, so the same code from SDK traces, promoted logs and external + * providers lands in one issue. Mirrored by the ingester and the backend. + * Returns null for an invalid code. + */ +export async function codeFingerprint( + service: string, + code: string, +): Promise { + if (!isValidErrorCode(code)) return null; + const subtle = (globalThis as { crypto?: { subtle?: SubtleCrypto } }).crypto + ?.subtle; + if (!subtle) throw new Error("codeFingerprint requires WebCrypto"); + const data = new TextEncoder().encode( + `${CODE_FINGERPRINT_SCHEME}\u0000${service}\u0000${code}`, + ); + const digest = new Uint8Array(await subtle.digest("SHA-256", data)); + let hex = ""; + for (const byte of digest) hex += byte.toString(16).padStart(2, "0"); + return hex.slice(0, 32); +} diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts new file mode 100644 index 0000000..6de6fbc --- /dev/null +++ b/packages/runtime-core/src/index.ts @@ -0,0 +1,91 @@ +/** + * @autter/runtime-core — PRIVATE. Bundled into @autter/runtime-node and + * @autter/runtime-edge at build time (tsup `noExternal`); never published + * and never a runtime dependency of a published package. Must stay free of + * Node built-ins: it runs on Workers, Vercel Edge, Deno and Bun. + */ +export type { Attributes, AttributeValue } from "./attributes.js"; +export { + redactAttributes, + makeRedactor, + type RedactOptions, +} from "./redact.js"; +export { + LEVEL_SEVERITY, + maxLevel, + boundContext, + userContext, + mergeContext, + toSpanAttributes, + type Redactor, + type RuntimeLogContext, + type RuntimeLogContextValue, + type RuntimeLogLevel, + type RuntimeOutcome, + type RuntimeOperationKind, +} from "./context.js"; +export { + CODE_PATTERN, + CODE_MAX_LENGTH, + RuntimeError, + defineRuntimeErrors, + isRuntimeErrorLike, + isValidErrorCode, + errorCodeOf, + errorStatusOf, + isExpectedError, + errorAttributes, + errorInternal, + toClientError, + responseStatusOf, + type RuntimeErrorDefinition, + type RuntimeErrorExtras, + type RuntimeErrorOptions, + type RuntimeErrorLike, + type RuntimeErrorFactory, + type RuntimeErrorCatalog, + type ClientErrorBody, + type ErrorAttributeOptions, +} from "./errors.js"; +export { + INLINE_LOG_LIMIT, + createInlineLogState, + appendInlineLog, + addAiUsage, + createRuntimeEvent, + isSummary, + severityOf, + type RuntimeEvent, + type InlineLog, + type InlineLogState, + type AiRollup, + type AiUsageInput, +} from "./record.js"; +export { + otlpValue, + toOtlpLogRecord, + buildOtlpLogsRequest, + type OtlpLogRecord, + type OtlpResource, +} from "./otlp-logs.js"; +export { + createCarrier, + parseCarrier, + formatTraceparent, + parseTraceparent, + type RuntimeCarrier, +} from "./carrier.js"; +export { + REQUEST_ID_PATTERN, + DEFAULT_REQUEST_ID_HEADER, + resolveRequestId, + normalizeRoutePath, + compileIgnore, + requestOutcome, + type RequestOutcomeInput, +} from "./request.js"; +export { parseUserAgent, type UserAgentInfo } from "./user-agent.js"; +export { + CODE_FINGERPRINT_SCHEME, + codeFingerprint, +} from "./fingerprint-code.js"; diff --git a/packages/runtime-core/src/otlp-logs.ts b/packages/runtime-core/src/otlp-logs.ts new file mode 100644 index 0000000..d53b080 --- /dev/null +++ b/packages/runtime-core/src/otlp-logs.ts @@ -0,0 +1,80 @@ +import type { RuntimeEvent } from "./record.js"; + +/** One OTLP/HTTP JSON log record, as runtime-node 1.4.0 encoded it. */ +export interface OtlpLogRecord { + timeUnixNano: string; + severityNumber: number; + severityText: string; + body: { stringValue: string }; + attributes: Array<{ key: string; value: Record }>; + traceId?: string; + spanId?: string; +} + +export interface OtlpResource { + service: string; + environment: string; + release?: string; +} + +const SEVERITY = { debug: 5, info: 9, warning: 13, error: 17 } as const; + +/** OTLP AnyValue encoding (bounded: 64 array items, 128 keys, 32 000 chars). */ +export function otlpValue(input: unknown): Record { + if (typeof input === "number") return { doubleValue: input }; + if (typeof input === "boolean") return { boolValue: input }; + if (Array.isArray(input)) + return { arrayValue: { values: input.slice(0, 64).map(otlpValue) } }; + if (input && typeof input === "object") + return { + kvlistValue: { + values: Object.entries(input) + .slice(0, 128) + .map(([key, item]) => ({ key, value: otlpValue(item) })), + }, + }; + return { stringValue: String(input ?? "").slice(0, 32000) }; +} + +/** Encode one event. Byte-compatible with runtime-node 1.4.0. */ +export function toOtlpLogRecord(event: RuntimeEvent): OtlpLogRecord { + return { + timeUnixNano: String(BigInt(Math.trunc(event.time)) * 1_000_000n), + severityNumber: SEVERITY[event.level], + severityText: event.level.toUpperCase(), + body: otlpValue(event.message) as { stringValue: string }, + attributes: Object.entries(event.attributes).map(([key, item]) => ({ + key, + value: otlpValue(item), + })), + ...(event.traceId + ? { traceId: event.traceId, spanId: event.spanId ?? "" } + : {}), + }; +} + +/** Wrap encoded records in an ExportLogsServiceRequest (scope "autter-runtime"). */ +export function buildOtlpLogsRequest( + resource: OtlpResource, + records: OtlpLogRecord[], +): Record { + return { + resourceLogs: [ + { + resource: { + attributes: Object.entries({ + "service.name": resource.service, + "deployment.environment.name": resource.environment, + ...(resource.release ? { "service.version": resource.release } : {}), + }).map(([key, item]) => ({ key, value: otlpValue(item) })), + }, + scopeLogs: [ + { + scope: { name: "autter-runtime" }, + logRecords: records, + }, + ], + }, + ], + }; +} diff --git a/packages/runtime-core/src/record.ts b/packages/runtime-core/src/record.ts new file mode 100644 index 0000000..96be92c --- /dev/null +++ b/packages/runtime-core/src/record.ts @@ -0,0 +1,194 @@ +import { + LEVEL_SEVERITY, + boundContext, + maxLevel, + type Redactor, + type RuntimeLogContext, + type RuntimeLogLevel, +} from "./context.js"; + +/** + * One runtime record before it is encoded for any sink: an operation or + * request summary (`autter.event.type = "operation"`) or a plain log line. + * `attributes` are already redacted and bounded. + */ +export interface RuntimeEvent { + /** Epoch milliseconds. */ + time: number; + level: RuntimeLogLevel; + message: string; + attributes: RuntimeLogContext; + traceId?: string; + spanId?: string; +} + +/** One message folded into an operation summary (`autter.operation.logs`). */ +export interface InlineLog { + /** Milliseconds since the operation started. */ + t: number; + level: RuntimeLogLevel; + message: string; + attrs?: RuntimeLogContext; +} + +/** `autter.operation.ai` — LLM usage accumulated by one operation. */ +export interface AiRollup { + calls: number; + input_tokens: number; + output_tokens: number; + cache_read_tokens: number; + cost_usd: number; + models: string[]; +} + +export interface AiUsageInput { + model?: string; + inputTokens?: number; + outputTokens?: number; + cacheReadTokens?: number; + costUsd?: number; +} + +export const INLINE_LOG_LIMIT = 50; +const INLINE_MESSAGE_MAX = 300; +const INLINE_CHARACTER_BUDGET = 6000; +const AI_MODEL_LIMIT = 10; + +/** Mutable inline-log state carried by an operation. */ +export interface InlineLogState { + startedAt: number; + logs: InlineLog[]; + logsTruncated: boolean; + logCharacters: number; + level: RuntimeLogLevel; +} + +export function createInlineLogState(startedAt: number): InlineLogState { + return { + startedAt, + logs: [], + logsTruncated: false, + logCharacters: 0, + level: "info", + }; +} + +/** + * Fold a message into an operation's timeline: max 50 entries, 300-char + * messages, small redacted attribute trees, ~6 KB in total. Overflow sets + * `logsTruncated`; the operation level is raised either way. + */ +export function appendInlineLog( + state: InlineLogState, + level: RuntimeLogLevel, + message: string, + attrs: RuntimeLogContext | undefined, + redact?: Redactor, + now = Date.now(), +): void { + state.level = maxLevel(state.level, level); + if (state.logs.length >= INLINE_LOG_LIMIT) { + state.logsTruncated = true; + return; + } + const text = String(boundContext({ message }, redact).message ?? "").slice( + 0, + INLINE_MESSAGE_MAX, + ); + let small: RuntimeLogContext | undefined; + if (attrs && Object.keys(attrs).length) { + small = boundContext(attrs, redact); + const encoded = JSON.stringify(small); + if (encoded.length > 1000) small = { "autter.context.truncated": true }; + } + const size = text.length + (small ? JSON.stringify(small).length : 0); + if (state.logCharacters + size > INLINE_CHARACTER_BUDGET) { + state.logsTruncated = true; + return; + } + state.logCharacters += size; + state.logs.push({ + t: Math.max(0, now - state.startedAt), + level, + message: text, + ...(small ? { attrs: small } : {}), + }); +} + +/** Add one LLM call to a rollup (creating it on first use). */ +export function addAiUsage( + rollup: AiRollup | undefined, + usage: AiUsageInput, +): AiRollup { + const next = rollup ?? { + calls: 0, + input_tokens: 0, + output_tokens: 0, + cache_read_tokens: 0, + cost_usd: 0, + models: [], + }; + const count = (value: number | undefined) => + typeof value === "number" && Number.isFinite(value) && value > 0 + ? Math.round(value) + : 0; + next.calls += 1; + next.input_tokens += count(usage.inputTokens); + next.output_tokens += count(usage.outputTokens); + next.cache_read_tokens += count(usage.cacheReadTokens); + if ( + typeof usage.costUsd === "number" && + Number.isFinite(usage.costUsd) && + usage.costUsd > 0 + ) + next.cost_usd = Math.round((next.cost_usd + usage.costUsd) * 1e8) / 1e8; + if ( + usage.model && + !next.models.includes(usage.model) && + next.models.length < AI_MODEL_LIMIT + ) + next.models.push(String(usage.model).slice(0, 100)); + return next; +} + +const TRACE_ID = /^[a-f0-9]{32}$/; +const SPAN_ID = /^[a-f0-9]{16}$/; + +/** Assemble a RuntimeEvent; invalid or all-zero trace ids are omitted. */ +export function createRuntimeEvent(input: { + level: RuntimeLogLevel; + message: string; + attributes: RuntimeLogContext; + time?: number; + traceId?: string; + spanId?: string; +}): RuntimeEvent { + const valid = + input.traceId !== undefined && + TRACE_ID.test(input.traceId) && + !/^0+$/.test(input.traceId); + return { + time: input.time ?? Date.now(), + level: input.level, + message: input.message, + attributes: input.attributes, + ...(valid + ? { + traceId: input.traceId, + ...(input.spanId && SPAN_ID.test(input.spanId) + ? { spanId: input.spanId } + : { spanId: input.spanId ?? "" }), + } + : {}), + }; +} + +/** True for operation/request summaries. */ +export function isSummary(event: RuntimeEvent): boolean { + return event.attributes["autter.event.type"] === "operation"; +} + +/** Severity number for a level (OTLP). */ +export function severityOf(level: RuntimeLogLevel): number { + return LEVEL_SEVERITY[level]; +} diff --git a/packages/runtime-core/src/redact.ts b/packages/runtime-core/src/redact.ts new file mode 100644 index 0000000..7e8a8a0 --- /dev/null +++ b/packages/runtime-core/src/redact.ts @@ -0,0 +1,409 @@ +import type { Attributes } from "./attributes.js"; + +/** + * Server-side attribute redaction. The browser relay whitelist-sanitises + * everything a client posts (see relay.ts) — but custom attributes handed to + * `captureException` / `captureMessage` left the process verbatim. This module + * closes that gap: emails, tokens, API keys, and credentials embedded in + * string values are masked, and attributes whose NAME looks sensitive are + * masked wholesale, before anything is exported. + */ + +type AttrValue = + | string + | number + | boolean + | Array + | object; + +export interface RedactOptions { + /** + * Extra patterns matched against lower-cased attribute KEYS; a match + * masks the whole value. Strings become case-insensitive substring + * patterns. Extends the built-in list. + */ + additionalKeyPatterns?: (RegExp | string)[]; + /** + * Extra patterns scrubbed inside string VALUES (same semantics as the + * built-in email/token patterns). Extends the built-in list. + */ + additionalValuePatterns?: (RegExp | string)[]; + /** Replacement token. Default "[redacted]". */ + mask?: string; + /** + * Scrub email-shaped substrings from ALL string values, not just + * sensitive keys. Default true. + */ + scrubEmailValues?: boolean; +} + +// Tested against the lower-cased KEY. Deliberately anchored where a loose +// substring would over-redact ("card" must not eat "discard", +// "author" must not eat "author_id"). +const SENSITIVE_KEY_PATTERNS: RegExp[] = [ + /e-?mail/, + /pass(word|wd|phrase)|^pass$/, + /token/, + /secret/, + /credential/, + /(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key/, + /^-?x?-?authorization$|^auth(-|_|$)|bearer/, + /cookie/, + /phone|msisdn/, + /\bssn\b|social[-_ ]?security/, + /cvv|cvc|card([-_. ]?(number|num|no))?$/, + /credit[-_.]?card/, + /connection[-_.]?string/, + /recovery[-_.]?code|\botp\b|magic[-_.]?link/, +]; + +// Scrubbed INSIDE string values (matched substrings are replaced, the rest +// of the value survives — useful context like a stack frame stays readable). +const PRIVATE_KEY_BLOCK_RE = + /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; +const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; +const OPENAI_KEY_RE = /\bsk-[A-Za-z0-9]{20,}\b/g; +const GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g; +const AWS_KEY_RE = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g; +const SLACK_TOKEN_RE = /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g; +const BEARER_RE = /\bbearer\s+[A-Za-z0-9._~+/=-]{10,}/gi; +// postgres://user:password@host — credentials gone, host kept. +const URL_CREDENTIALS_RE = + /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi; + +interface CompiledRedactor { + keyPatterns: RegExp[]; + valuePatterns: RegExp[]; + urlCredentials: RegExp; + privateKeyBlock: RegExp; + mask: string; + scrubEmailValues: boolean; +} + +function toCaseInsensitive(pattern: RegExp | string): RegExp { + return typeof pattern === "string" + ? new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i") + : new RegExp(pattern.source, pattern.flags.includes("i") ? pattern.flags : `${pattern.flags}i`); +} + +function compile(options?: RedactOptions): CompiledRedactor { + const keyPatterns = [...SENSITIVE_KEY_PATTERNS]; + const valuePatterns: RegExp[] = [ + JWT_RE, + OPENAI_KEY_RE, + GITHUB_TOKEN_RE, + AWS_KEY_RE, + SLACK_TOKEN_RE, + BEARER_RE, + ]; + if (options?.additionalKeyPatterns) { + for (const p of options.additionalKeyPatterns) { + keyPatterns.push(toCaseInsensitive(p)); + } + } + if (options?.additionalValuePatterns) { + for (const p of options.additionalValuePatterns) { + if (!valuePatterns.some((existing) => existing.source === (typeof p === "string" ? p : p.source))) { + valuePatterns.push(typeof p === "string" ? new RegExp(p, "gi") : p); + } + } + } + return { + keyPatterns, + valuePatterns, + urlCredentials: URL_CREDENTIALS_RE, + privateKeyBlock: PRIVATE_KEY_BLOCK_RE, + mask: options?.mask ?? "[redacted]", + scrubEmailValues: options?.scrubEmailValues !== false, + }; +} + +function redactString(value: string, r: CompiledRedactor): string { + let out = value.replace(r.privateKeyBlock, r.mask); + for (const re of r.valuePatterns) { + out = out.replace(re, r.mask); + } + out = out.replace(r.urlCredentials, "$1" + r.mask + "@"); + if (r.scrubEmailValues) { + out = out.replace(EMAIL_VALUE_RE, r.mask); + } + return out; +} + +const MAX_REDACTION_DEPTH = 64; +const MAX_REDACTION_WORK = 10_000; +const MAX_COLLECTION_ENTRIES = 1_000; + +interface RedactionState { + ancestors: WeakMap; + remainingWork: number; +} + +function canTraverse(depth: number, state: RedactionState): boolean { + if (depth > MAX_REDACTION_DEPTH || state.remainingWork <= 0) return false; + state.remainingWork -= 1; + return true; +} + +function redactValue( + value: unknown, + r: CompiledRedactor, + state: RedactionState, + depth: number, +): unknown { + if (typeof value === "string") return redactString(value, r); + + let isArray = false; + try { + isArray = Array.isArray(value); + } catch { + return r.mask; + } + + if (isArray) { + return redactArray(value as unknown[], r, state, depth); + } + + if (typeof value === "object" && value !== null) { + return redactObject(value, r, state, depth); + } + + return value; +} + +function redactArray( + value: unknown[], + r: CompiledRedactor, + state: RedactionState, + depth: number, +): unknown { + const existing = state.ancestors.get(value); + if (existing) return r.mask; + + if (!canTraverse(depth, state)) return r.mask; + + const out: unknown[] = []; + state.ancestors.set(value, out); + + let length: number; + try { + length = value.length; + } catch { + state.ancestors.delete(value); + return r.mask; + } + + const limit = Math.min(length, MAX_COLLECTION_ENTRIES); + + for (let i = 0; i < limit; i += 1) { + let item: unknown; + + try { + item = value[i]; + } catch { + out.push(r.mask); + continue; + } + + out.push(redactValue(item, r, state, depth + 1)); + } + + if (length > limit) { + out.push(r.mask); + } + + state.ancestors.delete(value); + return out; +} + +function redactObject( + value: object, + r: CompiledRedactor, + state: RedactionState, + depth: number, +): unknown { + const existing = state.ancestors.get(value); + if (existing) return r.mask; + + if (!canTraverse(depth, state)) return r.mask; + + const out: Record = {}; + state.ancestors.set(value, out); + + let count = 0; + let truncated = false; + + try { + for (const key in value as Record) { + if ( + !Object.prototype.propertyIsEnumerable.call( + value, + key, + ) + ) { + continue; + } + + if (count >= MAX_COLLECTION_ENTRIES) { + truncated = true; + break; + } + + let nestedValue: unknown; + try { + nestedValue = (value as Record)[key]; + } catch { + out[key] = r.mask; + count += 1; + continue; + } + + count += 1; + out[key] = isSensitiveKey(key, nestedValue, r) + ? r.mask + : redactValue( + nestedValue, + r, + state, + depth + 1, + ); + } + } catch { + truncated = true; + } + + if (truncated) { + out.__redaction_truncated__ = r.mask; + } + + state.ancestors.delete(value); + return out; +} +const USAGE_TOKEN_KEYS = new Set([ + "input_tokens", + "output_tokens", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "token_count", + "cache_read_tokens", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.prompt_tokens", + "gen_ai.usage.completion_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.usage.token_count", +]); + +function isSensitiveKey( + key: string, + value: unknown, + r: CompiledRedactor, +): boolean { + const lowered = key.toLowerCase(); + + // Canonical GenAI usage attributes are safe when they contain + // valid non-negative numeric counts. + if (USAGE_TOKEN_KEYS.has(lowered)) { + return !( + typeof value === "number" && + Number.isFinite(value) && + value >= 0 + ); + } + + // All other sensitive keys, including token-like keys, are redacted. + return r.keyPatterns.some((re) => re.test(lowered)); +} + +/** + * Return a copy of `attributes` with PII/secrets masked. Never mutates the + * input; non-string primitives pass through untouched; `undefined` values + * are dropped (OpenTelemetry rejects them). + */ +export function redactAttributes( + attributes?: Attributes | null, + options?: RedactOptions, +): Attributes { + const r = compile(options); + return redactWith(attributes, r); +} + +function redactWith( + attributes: Attributes | null | undefined, + r: CompiledRedactor, +): Attributes { + const out: Attributes = {}; + if (!attributes) return out; + + const state: RedactionState = { + ancestors: new WeakMap(), + remainingWork: MAX_REDACTION_WORK, + }; + + let count = 0; + let truncated = false; + + try { + for (const key in attributes as Record) { + if ( + !Object.prototype.propertyIsEnumerable.call( + attributes, + key, + ) + ) { + continue; + } + + if ( + count >= MAX_COLLECTION_ENTRIES || + state.remainingWork <= 0 + ) { + truncated = true; + break; + } + + let value: unknown; + try { + value = (attributes as Record)[key]; + } catch { + out[key] = r.mask; + count += 1; + state.remainingWork -= 1; + continue; + } + + count += 1; + state.remainingWork -= 1; + + if (value === undefined) continue; + + out[key] = isSensitiveKey(key, value, r) + ? r.mask + : (redactValue(value, r, state, 0) as Attributes[string]); + } + } catch { + truncated = true; + } + + if (truncated) { + out.__redaction_truncated__ = r.mask; + } + + return out; +} + +/** + * Compile a redactor once for a hot path — initAutterServer builds one from + * its options and reuses it for every capture instead of recompiling. + */ +export function makeRedactor( + options?: boolean | RedactOptions, +): (attributes?: Attributes | null) => Attributes { + if (options === false) { + return (attributes) => ({ ...(attributes ?? {}) }); + } + const r = compile(options === true ? undefined : options); + return (attributes) => redactWith(attributes, r); +} diff --git a/packages/runtime-core/src/request.ts b/packages/runtime-core/src/request.ts new file mode 100644 index 0000000..489859c --- /dev/null +++ b/packages/runtime-core/src/request.ts @@ -0,0 +1,95 @@ +import type { RuntimeOutcome } from "./context.js"; +import { isExpectedError } from "./errors.js"; + +/** Honoured inbound request ids: 8–128 word characters, dots or dashes. */ +export const REQUEST_ID_PATTERN = /^[\w.-]{8,128}$/; +export const DEFAULT_REQUEST_ID_HEADER = "x-request-id"; + +function randomId(): string { + const c = (globalThis as { crypto?: { randomUUID?: () => string } }).crypto; + if (c?.randomUUID) return c.randomUUID(); + // Last resort for exotic runtimes without WebCrypto. + return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (d) => + (Number(d) ^ ((Math.random() * 16) >> (Number(d) / 4))).toString(16), + ); +} + +/** The inbound id when it matches REQUEST_ID_PATTERN, else a fresh UUID. */ +export function resolveRequestId(header: unknown): string { + const value = Array.isArray(header) ? header[0] : header; + return typeof value === "string" && REQUEST_ID_PATTERN.test(value.trim()) + ? value.trim() + : randomId(); +} + +/** Replace id-like path segments (UUIDs, numbers, long hex) with `:id` — + * the same rules the ingester uses for route grouping. Query strings dropped. */ +export function normalizeRoutePath(path: string): string { + const pathname = (path.split(/[?#]/)[0] ?? "") || "/"; + return pathname + .split("/") + .map((segment) => { + if (!segment) return segment; + if ( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + segment, + ) + ) + return ":id"; + if (/^\d+$/.test(segment)) return ":id"; + if (/^[0-9a-f]{8,}$/i.test(segment)) return ":id"; + return segment; + }) + .join("/") + .slice(0, 200); +} + +/** + * Compile path globs: `*` matches within one segment, `**` across + * segments; a pattern without wildcards matches the exact path. + */ +export function compileIgnore( + patterns: readonly string[] = [], +): (path: string) => boolean { + const regexes = patterns.map((pattern) => { + let source = ""; + for (let i = 0; i < pattern.length; i++) { + const char = pattern[i]!; + if (char === "*" && pattern[i + 1] === "*") { + source += ".*"; + i++; + } else if (char === "*") source += "[^/]*"; + else source += char.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); + } + return new RegExp(`^${source}/?$`); + }); + return (path) => { + const pathname = path.split(/[?#]/)[0] ?? ""; + return regexes.some((re) => re.test(pathname)); + }; +} + +export interface RequestOutcomeInput { + /** Outcome set explicitly via `outcome()` — always wins. */ + explicit?: RuntimeOutcome; + /** Error recorded on the request (thrown, error middleware, capture). */ + error?: unknown; + /** The error escaped the handler (vs. captured and recovered). */ + thrown?: boolean; + status?: number; + aborted?: boolean; +} + +/** + * Explicit outcome wins. Otherwise an expected coded error → degraded, a + * thrown error or status ≥ 500 → failed, an aborted request → cancelled, + * anything else → succeeded. + */ +export function requestOutcome(input: RequestOutcomeInput): RuntimeOutcome { + if (input.explicit) return input.explicit; + if (input.error !== undefined && isExpectedError(input.error)) return "degraded"; + if (input.thrown || (input.status !== undefined && input.status >= 500)) + return "failed"; + if (input.aborted) return "cancelled"; + return "succeeded"; +} diff --git a/packages/runtime-core/src/user-agent.ts b/packages/runtime-core/src/user-agent.ts new file mode 100644 index 0000000..75c7b92 --- /dev/null +++ b/packages/runtime-core/src/user-agent.ts @@ -0,0 +1,55 @@ +/** + * Coarse user-agent classification — the same browser/OS mapping as + * @autter/runtime-browser (family + major version, OS family), plus a + * device class. Never stores the raw user agent. + */ +export interface UserAgentInfo { + browser: string; + os: string; + device: "desktop" | "mobile" | "tablet" | "bot" | "unknown"; + bot: boolean; +} + +const BOT = + /bot\b|crawler|spider|crawling|slurp|headless|lighthouse|curl\/|wget\/|python-requests|httpclient|okhttp|go-http-client|axios\/|node-fetch|undici/i; + +export function parseUserAgent(ua: string | undefined | null): UserAgentInfo { + const value = String(ua ?? "").slice(0, 512); + const os = /Windows/.test(value) + ? "Windows" + : /Android/.test(value) + ? "Android" + : /iPhone|iPad|iPod/.test(value) + ? "iOS" + : /Mac OS X/.test(value) + ? "macOS" + : /CrOS/.test(value) + ? "ChromeOS" + : /Linux/.test(value) + ? "Linux" + : ""; + const edge = /Edg\/(\d+)/.exec(value); + const firefox = /Firefox\/(\d+)/.exec(value); + const chrome = /Chrome\/(\d+)/.exec(value); + const safari = /Version\/(\d+).+Safari/.exec(value); + const browser = edge + ? `Edge ${edge[1]}` + : firefox + ? `Firefox ${firefox[1]}` + : chrome + ? `Chrome ${chrome[1]}` + : safari + ? `Safari ${safari[1]}` + : ""; + const bot = BOT.test(value); + const device = !value + ? "unknown" + : bot + ? "bot" + : /iPad|Tablet/.test(value) || (/Android/.test(value) && !/Mobile/.test(value)) + ? "tablet" + : /Mobi|iPhone|iPod|Android/.test(value) + ? "mobile" + : "desktop"; + return { browser, os, device, bot }; +} diff --git a/packages/runtime-core/test/core.test.mjs b/packages/runtime-core/test/core.test.mjs new file mode 100644 index 0000000..e53a772 --- /dev/null +++ b/packages/runtime-core/test/core.test.mjs @@ -0,0 +1,233 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { + RuntimeError, + defineRuntimeErrors, + isRuntimeErrorLike, + errorAttributes, + errorInternal, + toClientError, + responseStatusOf, + isValidErrorCode, + codeFingerprint, + parseCarrier, + createCarrier, + resolveRequestId, + normalizeRoutePath, + compileIgnore, + requestOutcome, + createInlineLogState, + appendInlineLog, + addAiUsage, + parseUserAgent, + toOtlpLogRecord, + boundContext, + mergeContext, +} from "../dist/index.js"; + +test("code fingerprints match the shared code-v1 test vectors", async () => { + const vectors = JSON.parse( + readFileSync(new URL("../fixtures/code-fingerprints.json", import.meta.url)), + ); + for (const { service, code, fingerprint } of vectors) + assert.equal(await codeFingerprint(service, code), fingerprint); + assert.equal(await codeFingerprint("svc", "Bad Code"), null); +}); + +test("code validation follows CODE_PATTERN and the 80-char limit", () => { + for (const ok of ["billing", "billing.declined", "a.b.c.d", "github.app_permission_missing"]) + assert.ok(isValidErrorCode(ok), ok); + for (const bad of ["Billing.x", "1abc", "a.b.c.d.e", "billing-declined", "a.", `a${"x".repeat(80)}`, "ECONNREFUSED"]) + assert.ok(!isValidErrorCode(bad), bad); +}); + +test("catalogs build typed, namespaced RuntimeErrors", () => { + const billing = defineRuntimeErrors("billing", { + declined: { + status: 402, + message: "Payment declined", + expected: true, + why: "The card issuer rejected the charge", + fix: "Ask for another card", + link: "https://docs.example.com/payments#declined", + }, + limit: ({ plan }) => ({ status: 429, message: `Plan ${plan} limit reached`, fix: "Upgrade" }), + }); + const declined = billing.declined({ internal: { cardBin: "424242" } }); + assert.ok(declined instanceof RuntimeError); + assert.equal(declined.code, "billing.declined"); + assert.equal(billing.declined.code, "billing.declined"); + assert.equal(declined.status, 402); + assert.equal(declined.expected, true); + const limit = billing.limit({ plan: "free" }); + assert.equal(limit.code, "billing.limit"); + assert.equal(limit.message, "Plan free limit reached"); + assert.match(limit.stack, /core\.test\.mjs/); + // internal is not enumerable, never serialised + assert.equal(JSON.stringify(declined).includes("424242"), false); + assert.deepEqual(errorInternal(declined), { cardBin: "424242" }); +}); + +test("invalid codes are dropped with a one-time warning", () => { + const warnings = []; + const original = console.warn; + console.warn = (message) => warnings.push(message); + try { + const a = new RuntimeError({ code: "Order 1234 failed", message: "x" }); + new RuntimeError({ code: "Order 1234 failed", message: "y" }); + assert.equal(a.code, undefined); + assert.equal(warnings.length, 1); + assert.deepEqual(errorAttributes(a), {}); + } finally { + console.warn = original; + } +}); + +test("duck typing reads code/why/fix/link/status/expected from any error", () => { + class AppError extends Error { + constructor() { + super("Session expired"); + this.code = "auth.session_expired"; + this.why = "Token older than 24h"; + this.statusCode = 401; + this.expected = true; + } + } + const err = new AppError(); + assert.ok(isRuntimeErrorLike(err)); + assert.ok(!isRuntimeErrorLike(new Error("plain"))); + assert.deepEqual(errorAttributes(err), { + "autter.error.code": "auth.session_expired", + "autter.error.why": "Token older than 24h", + "autter.error.status": 401, + "autter.error.expected": true, + }); + // foreign codes are ignored, not warned + const sys = Object.assign(new Error("connect"), { code: "ECONNREFUSED" }); + assert.equal(errorAttributes(sys)["autter.error.code"], undefined); +}); + +test("cause chains become exception.cause.N.* (max 5)", () => { + let cause = Object.assign(new Error("root"), { code: "ECONNRESET" }); + for (let i = 0; i < 7; i++) cause = new Error(`level ${i}`, { cause }); + const err = new RuntimeError({ code: "inventory.reservation_timeout", message: "timeout", cause }); + const attrs = errorAttributes(err); + assert.equal(attrs["exception.cause.1.message"], "level 6"); + assert.equal(attrs["exception.cause.5.message"], "level 2"); + assert.equal(attrs["exception.cause.6.message"], undefined); + const short = errorAttributes(new RuntimeError({ message: "x", cause: Object.assign(new TypeError("t"), { code: "E1" }) })); + assert.equal(short["exception.cause.1.type"], "TypeError"); + assert.equal(short["exception.cause.1.code"], "E1"); +}); + +test("toClientError never leaks internal or plain 5xx messages", () => { + const err = new RuntimeError({ + code: "billing.limit", + message: "Plan limit reached", + status: 429, + fix: "Upgrade", + internal: { secret: "s3cr3t" }, + cause: new Error("db password=hunter2"), + }); + const body = toClientError(err, "req-12345678"); + assert.deepEqual(body, { + error: { message: "Plan limit reached", code: "billing.limit", fix: "Upgrade", requestId: "req-12345678" }, + }); + assert.equal(JSON.stringify(body).includes("s3cr3t"), false); + assert.equal(JSON.stringify(body).includes("hunter2"), false); + assert.deepEqual(toClientError(new Error("SELECT * FROM users failed")), { + error: { message: "Internal Server Error" }, + }); + assert.equal(responseStatusOf(err), 429); + assert.equal(responseStatusOf(new Error("x")), 500); +}); + +test("carriers round-trip and reject malformed input", () => { + const carrier = createCarrier({ + op: "6f1c2b9e-1111-4222-8333-944455556666", + req: "req-abcdef12", + traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + }); + assert.deepEqual(parseCarrier(JSON.stringify(carrier)), carrier); + assert.equal(parseCarrier({ v: 2, op: "x" }), null); + assert.equal(parseCarrier("not json"), null); + assert.deepEqual(parseCarrier({ v: 1, op: "op-1", req: "bad id!", traceparent: "junk" }), { v: 1, op: "op-1" }); +}); + +test("request helpers: ids, route normalisation, ignore globs, outcomes", () => { + assert.equal(resolveRequestId("abc12345"), "abc12345"); + assert.match(resolveRequestId("short"), /^[0-9a-f-]{36}$/); + assert.match(resolveRequestId("has spaces in it"), /^[0-9a-f-]{36}$/); + assert.equal(normalizeRoutePath("/orders/812/items/6f1c2b9e-1111-4222-8333-944455556666?x=1"), "/orders/:id/items/:id"); + const ignored = compileIgnore(["/healthz", "/metrics/*", "/static/**"]); + assert.ok(ignored("/healthz")); + assert.ok(ignored("/healthz?probe=1")); + assert.ok(ignored("/metrics/node")); + assert.ok(!ignored("/metrics/node/extra")); + assert.ok(ignored("/static/a/b/c.js")); + assert.ok(!ignored("/api")); + assert.equal(requestOutcome({ status: 200 }), "succeeded"); + assert.equal(requestOutcome({ status: 503 }), "failed"); + assert.equal(requestOutcome({ thrown: true, status: 200 }), "failed"); + assert.equal(requestOutcome({ aborted: true, status: 200 }), "cancelled"); + assert.equal(requestOutcome({ error: { expected: true }, thrown: true, status: 402 }), "degraded"); + assert.equal(requestOutcome({ explicit: "degraded", status: 500 }), "degraded"); +}); + +test("inline logs are bounded at 50 and track the highest level", () => { + const state = createInlineLogState(Date.now()); + appendInlineLog(state, "info", "hello person@example.com", { token: "abc" }); + assert.equal(state.logs[0].message.includes("person@example.com"), false); + assert.equal(state.logs[0].attrs.token, "[redacted]"); + appendInlineLog(state, "warning", "careful"); + for (let i = 0; i < 60; i++) appendInlineLog(state, "debug", `m${i}`); + assert.equal(state.logs.length, 50); + assert.equal(state.logsTruncated, true); + assert.equal(state.level, "warning"); +}); + +test("AI rollup accumulates usage and distinct models", () => { + let ai = addAiUsage(undefined, { model: "gpt-5-mini", inputTokens: 10, outputTokens: 5, costUsd: 0.001 }); + ai = addAiUsage(ai, { model: "gpt-5-mini", inputTokens: 1, cacheReadTokens: 4, costUsd: 0.002 }); + ai = addAiUsage(ai, { model: "claude-sonnet-4" }); + assert.deepEqual(ai, { + calls: 3, + input_tokens: 11, + output_tokens: 5, + cache_read_tokens: 4, + cost_usd: 0.003, + models: ["gpt-5-mini", "claude-sonnet-4"], + }); +}); + +test("user agents map coarsely", () => { + const chrome = parseUserAgent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + ); + assert.deepEqual(chrome, { browser: "Chrome 120", os: "Windows", device: "desktop", bot: false }); + assert.equal(parseUserAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) Version/17.0 Mobile Safari/604.1").device, "mobile"); + assert.equal(parseUserAgent("Googlebot/2.1 (+http://www.google.com/bot.html)").device, "bot"); + assert.equal(parseUserAgent("").device, "unknown"); +}); + +test("context bounding redacts, strips reserved keys and orders logs last", () => { + const merged = mergeContext({}, { + "autter.operation.id": "spoof", + "autter.request.id": "spoof", + "autter.error.code": "spoof.code", + password: "hunter2", + cart: { items: 2 }, + }); + assert.deepEqual(merged, { password: "[redacted]", cart: { items: 2 } }); + const bounded = boundContext({ user: 1, "autter.operation.logs": [], "autter.event.type": "operation", "exception.type": "E" }); + assert.deepEqual(Object.keys(bounded), ["autter.event.type", "exception.type", "user", "autter.operation.logs"]); + const record = toOtlpLogRecord({ time: 1, level: "warning", message: "m", attributes: { a: 1 } }); + assert.deepEqual(record, { + timeUnixNano: "1000000", + severityNumber: 13, + severityText: "WARNING", + body: { stringValue: "m" }, + attributes: [{ key: "a", value: { doubleValue: 1 } }], + }); +}); diff --git a/packages/runtime-core/tsconfig.json b/packages/runtime-core/tsconfig.json new file mode 100644 index 0000000..5e4e2f6 --- /dev/null +++ b/packages/runtime-core/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022", "DOM"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"] +} diff --git a/packages/runtime-edge/README.md b/packages/runtime-edge/README.md new file mode 100644 index 0000000..c89cbbc --- /dev/null +++ b/packages/runtime-edge/README.md @@ -0,0 +1,117 @@ +# @autter/runtime-edge + +Autter Runtime for fetch-only runtimes — **Cloudflare Workers, Vercel Edge +(and Next.js `middleware.ts`), Deno and Bun**. Zero dependencies, no Node APIs, +no AsyncLocalStorage. + +Each request emits one wide **request summary** (method, route, status, +outcome, duration, request id, your context, inline messages, coded error), +always kept. Exceptions become issues; errors with the same code group into +one issue across your Node services, edge functions and external providers. + +Requires `@autter/otlp-ingester` **1.5.0+** and a **server** ingest key +(`autter_rt_…`) stored as a secret — never a browser/client key. + +## Install + +```bash +npm install @autter/runtime-edge +``` + +## Cloudflare Workers + +```ts +import { withAutter, defineRuntimeErrors } from "@autter/runtime-edge"; + +const billing = defineRuntimeErrors("billing", { + declined: { status: 402, message: "Payment declined", expected: true, + fix: "Ask the customer for another card" }, +}); + +export default withAutter( + // Bindings exist per request, so options may be a function of env. + (env) => ({ apiKey: env.AUTTER_RUNTIME_KEY, service: "edge-api", release: env.GIT_SHA }), + async (request, env, ctx, rt) => { + rt.set({ tenant: env.TENANT, colo: request.cf?.colo }); + rt.info("Routing request"); // folded into the summary + if (!(await authorise(request))) throw billing.declined(); + return new Response("ok"); + }, +); +``` + +`npx wrangler secret put AUTTER_RUNTIME_KEY`. Delivery runs through +`ctx.waitUntil`, so it never delays the response. + +## Vercel Edge / Next.js middleware + +```ts +// middleware.ts +import { NextResponse } from "next/server"; +import { withAutter } from "@autter/runtime-next/edge"; // or "@autter/runtime-edge" + +export default withAutter( + { apiKey: process.env.AUTTER_RUNTIME_KEY, service: "web-middleware" }, + async (request, _event, _ctx, rt) => NextResponse.next(), +); +``` + +The `NextFetchEvent` (second argument) supplies `waitUntil`. + +## Deno and Bun + +```ts +const handler = withAutter({ apiKey: Deno.env.get("AUTTER_RUNTIME_KEY"), service: "deno-api" }, + async (request, _env, _ctx, rt) => new Response("ok")); +Deno.serve(handler); // Bun: Bun.serve({ fetch: handler }) +``` + +Without `waitUntil`, delivery is fire-and-forget (the process stays up). +`await handler.flush()` delivers everything buffered. + +## The `rt` handle + +| Member | | +| --- | --- | +| `rt.set(context)` | Deep-merge redacted context into the request summary | +| `rt.outcome(status, message?)` | `succeeded` / `failed` / `degraded` / `cancelled` / `pending` — wins over the automatic rules | +| `rt.info(message, attrs?)` | Folded into the summary timeline (`autter.operation.logs`, max 50) | +| `rt.warn(message, attrs?)` | Folded **and** exported as its own record | +| `rt.error(err, attrs?)` | Like `warn` at error level; attaches the error's code/why/fix to the summary | +| `rt.captureException(err, attrs?)` | Report an exception — becomes an occurrence (`autter.capture.mode = "log"`) | +| `rt.requestId` | Honoured `x-request-id` (`^[\w.-]{8,128}$`) or a fresh UUID; echoed in the response header | + +Errors thrown from the handler are captured, recorded on the summary and +rethrown — or answered as `{ error: { message, code?, why?, fix?, link?, +requestId? } }` with `errorResponse: true`. Outcome rules: explicit outcome, +then an `expected` coded error → `degraded`, a thrown error or status ≥ 500 → +`failed`, an aborted request → `cancelled`, else `succeeded`. + +## Options + +| Option | Default | | +| --- | --- | --- | +| `apiKey` | — | Server key. Without it nothing is exported (one warning). | +| `endpoint` | `https://otlp.autter.dev` | Ingester base URL (`/v1/logs` is appended) | +| `service`, `environment`, `release` | —, `production`, — | Resource attributes | +| `requestIdHeader` | `x-request-id` | Also added to `Access-Control-Expose-Headers` when the response has CORS headers | +| `ignore` | `[]` | Path globs not summarised (`*` one segment, `**` any depth) | +| `routeOf(request)` | pathname with ids → `:id` | Route template for `http.route` and the summary name | +| `errorResponse` | `false` | Answer thrown errors with `toClientError` JSON | +| `minLevel` | `debug` | Drop plain messages below this level (summaries are always kept) | +| `console` | `false` | Also print records as JSON lines | +| `redactAttributes` | `true` | Same redaction as runtime-node (emails, tokens, sensitive keys) | +| `maxQueue` | `200` | Records buffered per isolate (1 MiB cap); overflow is dropped | + +Also exported: `RuntimeError`, `defineRuntimeErrors`, `isRuntimeErrorLike`, +`toClientError`, `CODE_PATTERN` — the same coded-error API as +`@autter/runtime-node`, so one catalog module can serve both. + +## Limits + +No tracing: records carry no trace ids, and there are no steps or child +operations. Each export makes at most two attempts; undelivered records are +dropped with a console warning. Codes must match +`^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$` (≤ 80 chars) — never ids or user data. + +See [Requests, coded errors and background work](../../docs/REQUESTS-AND-ERRORS.md). diff --git a/packages/runtime-edge/package.json b/packages/runtime-edge/package.json new file mode 100644 index 0000000..de84407 --- /dev/null +++ b/packages/runtime-edge/package.json @@ -0,0 +1,54 @@ +{ + "name": "@autter/runtime-edge", + "version": "1.0.0", + "description": "Autter Runtime for edge runtimes (Cloudflare Workers, Vercel Edge, Deno, Bun): request wide events and coded errors over fetch, zero dependencies", + "license": "MIT", + "type": "module", + "sideEffects": false, + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "https://github.com/Autter-dev/autter-runtime.git", + "directory": "packages/runtime-edge" + }, + "scripts": { + "build": "tsup", + "test": "npm run build && node --test test/*.test.mjs" + }, + "devDependencies": { + "@autter/runtime-core": "0.0.0", + "tsup": "^8.3.5", + "typescript": "^5.7.0" + }, + "engines": { + "node": ">=20" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "homepage": "https://github.com/Autter-dev/autter-runtime#readme", + "bugs": "https://github.com/Autter-dev/autter-runtime/issues", + "keywords": [ + "edge", + "cloudflare-workers", + "vercel-edge", + "deno", + "bun", + "error-tracking", + "telemetry", + "otlp" + ] +} diff --git a/packages/runtime-edge/src/index.ts b/packages/runtime-edge/src/index.ts new file mode 100644 index 0000000..c2634b1 --- /dev/null +++ b/packages/runtime-edge/src/index.ts @@ -0,0 +1,477 @@ +/** + * @autter/runtime-edge — Autter Runtime for fetch-only runtimes: Cloudflare + * Workers, Vercel Edge (and Next.js `middleware.ts` via + * `@autter/runtime-next/edge`), Deno and Bun. Zero dependencies, no + * AsyncLocalStorage: the request context is passed to your handler as `rt`. + * + * export default withAutter( + * (env) => ({ apiKey: env.AUTTER_RUNTIME_KEY, service: "edge-api" }), + * async (request, env, ctx, rt) => { + * rt.set({ tenant: env.TENANT }); + * if (!ok) throw billingErrors.declined(); + * return new Response("ok"); + * }, + * ); + * + * Each request emits one `kind: "request"` summary (always kept) plus + * warn/error records and captured exceptions, exported as OTLP/HTTP JSON to + * `${endpoint}/v1/logs` with a SERVER key. Delivery runs through + * `ctx.waitUntil` so it never delays the response. Exceptions are error + * records with `autter.capture.mode = "log"`, which the ingester (1.5.0+) + * promotes to occurrences. + */ +import { + DEFAULT_REQUEST_ID_HEADER, + appendInlineLog, + boundContext, + buildOtlpLogsRequest, + compileIgnore, + createInlineLogState, + createRuntimeEvent, + errorAttributes, + makeRedactor, + maxLevel, + mergeContext, + normalizeRoutePath, + requestOutcome, + resolveRequestId, + responseStatusOf, + toClientError, + toOtlpLogRecord, + userContext, + type OtlpLogRecord, + type RedactOptions, + type RuntimeLogContext, + type RuntimeLogLevel, + type RuntimeOutcome, +} from "@autter/runtime-core"; + +export { + CODE_PATTERN, + RuntimeError, + defineRuntimeErrors, + isRuntimeErrorLike, + toClientError, + type RuntimeErrorDefinition, + type RuntimeErrorExtras, + type RuntimeErrorOptions, + type RuntimeErrorLike, + type RuntimeErrorFactory, + type RuntimeErrorCatalog, + type ClientErrorBody, + type RedactOptions, + type RuntimeLogContext, + type RuntimeLogContextValue, + type RuntimeLogLevel, + type RuntimeOutcome, +} from "@autter/runtime-core"; + +export interface AutterEdgeOptions { + /** SERVER ingest key (autter_rt_…) from a secret binding. Omit to disable export. */ + apiKey?: string; + /** Ingester base URL. Default https://otlp.autter.dev */ + endpoint?: string; + service: string; + environment?: string; + release?: string; + /** Request id header to honour and echo. Default "x-request-id". */ + requestIdHeader?: string; + /** Path globs never summarised (`*` = one segment, `**` = any depth). */ + ignore?: string[]; + /** Route template for `http.route` (default: pathname with ids → `:id`). */ + routeOf?: (request: Request) => string | undefined; + /** Answer thrown errors with `toClientError` JSON instead of rethrowing. Default false. */ + errorResponse?: boolean; + /** Drop plain records below this level (summaries always kept). */ + minLevel?: RuntimeLogLevel; + /** Also print records as JSON lines. Default false. */ + console?: boolean; + /** Same semantics as runtime-node. Default true. */ + redactAttributes?: boolean | RedactOptions; + /** Max records buffered per isolate. Default 200. */ + maxQueue?: number; + /** @internal Override fetch (tests). */ + fetch?: typeof fetch; +} + +/** The per-request handle passed to your handler. */ +export interface EdgeRuntime { + /** Merge attributes into the request summary's context. */ + set(context: RuntimeLogContext): void; + /** Set the outcome explicitly (wins over status/error rules). */ + outcome(status: RuntimeOutcome, message?: string): void; + /** Folded into the summary timeline. */ + info(message: string, attributes?: RuntimeLogContext): void; + /** Folded into the summary AND emitted as its own record. */ + warn(message: string, attributes?: RuntimeLogContext): void; + /** Logs an error (folded + emitted) and attaches its code to the summary. */ + error(error: unknown, attributes?: RuntimeLogContext): void; + /** Request id (honoured `x-request-id` or a fresh UUID); echoed in the response. */ + readonly requestId: string; + /** Report an exception (becomes an occurrence/issue). */ + captureException(error: unknown, attributes?: RuntimeLogContext): void; +} + +export interface EdgeExecutionContext { + waitUntil(promise: Promise): void; +} + +export type EdgeHandler = ( + request: Request, + env: Env, + ctx: Ctx, + rt: EdgeRuntime, +) => Response | Promise; + +/** Callable handler that is also a Workers/Bun-style `{ fetch }` object. */ +export type AutterEdgeHandler = (( + request: Request, + env?: Env, + ctx?: Ctx, +) => Promise) & { + fetch(request: Request, env?: Env, ctx?: Ctx): Promise; + /** Deliver everything buffered now (normally done via waitUntil). */ + flush(): Promise; +}; + +interface Exporter { + queue: OtlpLogRecord[]; + bytes: number; + dropped: number; + flushing: Promise | null; +} + +const encoder = new TextEncoder(); + +function randomId(): string { + return globalThis.crypto.randomUUID(); +} + +function resolveOptions( + options: AutterEdgeOptions | ((env: Env) => AutterEdgeOptions), + env: Env, +): AutterEdgeOptions { + return typeof options === "function" ? options(env) : options; +} + +function exposeValue(current: string | null, header: string): string | null { + const names = (current ?? "") + .split(",") + .map((item) => item.trim().toLowerCase()) + .filter(Boolean); + if (names.includes("*") || names.includes(header)) return null; + return names.length ? `${current}, ${header}` : header; +} + +function withRequestId(response: Response, header: string, requestId: string): Response { + const apply = (target: Response) => { + target.headers.set(header, requestId); + if (target.headers.get("access-control-allow-origin") !== null) { + const value = exposeValue(target.headers.get("access-control-expose-headers"), header); + if (value) target.headers.set("access-control-expose-headers", value); + } + return target; + }; + try { + return apply(response); + } catch { + try { + return apply(new Response(response.body, response)); + } catch { + return response; + } + } +} + +/** + * Wrap an edge fetch handler. `options` may be a function of `env` (Workers + * bindings are only available per request). + */ +export function withAutter( + options: AutterEdgeOptions | ((env: Env) => AutterEdgeOptions), + handler: EdgeHandler, +): AutterEdgeHandler { + // Per isolate: buffered records survive between requests until flushed. + const exporter: Exporter = { queue: [], bytes: 0, dropped: 0, flushing: null }; + let lastOptions: AutterEdgeOptions | null = null; + let warnedNoKey = false; + + const deliver = async (opts: AutterEdgeOptions): Promise => { + if (exporter.flushing) await exporter.flushing.catch(() => {}); + if (!opts.apiKey || !exporter.queue.length) return; + const endpoint = (opts.endpoint ?? "https://otlp.autter.dev").replace(/\/$/, ""); + const doFetch = opts.fetch ?? globalThis.fetch.bind(globalThis); + exporter.flushing = (async () => { + while (exporter.queue.length) { + const batch = exporter.queue.splice(0, 50); + for (const record of batch) exporter.bytes -= encoder.encode(JSON.stringify(record)).length; + const body = JSON.stringify( + buildOtlpLogsRequest( + { + service: opts.service, + environment: opts.environment ?? "production", + ...(opts.release ? { release: opts.release } : {}), + }, + batch, + ), + ); + let ok = false; + for (let attempt = 0; attempt < 2 && !ok; attempt++) { + try { + const response = await doFetch(`${endpoint}/v1/logs`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${opts.apiKey}`, + }, + body, + }); + ok = response.ok; + // 4xx (bad key, payload rejected) will not improve on retry. + if (!ok && response.status < 500) break; + } catch { + /* retry once */ + } + } + if (!ok) { + exporter.dropped += batch.length; + console.warn( + `[autter-runtime-edge] ${batch.length} record(s) could not be delivered to ${endpoint}/v1/logs`, + ); + } + } + })().finally(() => { + exporter.flushing = null; + }); + return exporter.flushing; + }; + + const wrapped = async (request: Request, env?: Env, ctx?: Ctx): Promise => { + const opts = resolveOptions(options, env as Env); + lastOptions = opts; + const header = (opts.requestIdHeader ?? DEFAULT_REQUEST_ID_HEADER).toLowerCase(); + const redact = makeRedactor(opts.redactAttributes ?? true); + let pathname = "/"; + try { + pathname = new URL(request.url).pathname; + } catch { + /* keep "/" */ + } + const runtime = buildRuntime(request, pathname, opts, header, redact); + if (compileIgnore(opts.ignore)(pathname)) + return handler(request, env as Env, ctx as Ctx, runtime.rt); + if (!opts.apiKey && !warnedNoKey) { + warnedNoKey = true; + console.warn("[autter-runtime-edge] no apiKey: records are not exported"); + } + let status = 500; + let thrown: unknown; + let response: Response | undefined; + try { + response = await handler(request, env as Env, ctx as Ctx, runtime.rt); + status = response.status; + } catch (error) { + thrown = error; + runtime.noteError(error, true); + runtime.rt.captureException(error); + if (opts.errorResponse) { + status = responseStatusOf(error); + response = new Response(JSON.stringify(toClientError(error, runtime.rt.requestId)), { + status, + headers: { "content-type": "application/json; charset=utf-8" }, + }); + } + } + const records = runtime.finish(status, request.signal?.aborted === true); + for (const record of records) { + const encoded = toOtlpLogRecord(record); + const bytes = encoder.encode(JSON.stringify(encoded)).length; + if ( + exporter.queue.length >= (opts.maxQueue ?? 200) || + exporter.bytes + bytes > 1024 * 1024 || + bytes > 256 * 1024 + ) { + exporter.dropped++; + continue; + } + exporter.queue.push(encoded); + exporter.bytes += bytes; + if (opts.console) + console.log(JSON.stringify({ ...record.attributes, level: record.level, message: record.message })); + } + const flush = deliver(opts).catch(() => {}); + const waitUntil = + (ctx as Partial | undefined)?.waitUntil ?? + (env as Partial | undefined)?.waitUntil; + if (typeof waitUntil === "function") { + try { + waitUntil.call( + (ctx as Partial | undefined)?.waitUntil ? ctx : env, + flush, + ); + } catch { + /* best effort */ + } + } + if (thrown !== undefined && !response) throw thrown; + return withRequestId(response!, header, runtime.rt.requestId); + }; + return Object.assign(wrapped, { + fetch: wrapped, + flush: () => (lastOptions ? deliver(lastOptions) : Promise.resolve()), + }); +} + +/** Explicit per-request state (no AsyncLocalStorage on the edge). */ +function buildRuntime( + request: Request, + pathname: string, + opts: AutterEdgeOptions, + header: string, + redact: ReturnType, +) { + const startedAt = Date.now(); + const requestId = resolveRequestId(request.headers.get(header)); + const method = String(request.method || "GET").toUpperCase(); + const route = (opts.routeOf?.(request) ?? normalizeRoutePath(pathname)).slice(0, 200); + const operationId = randomId(); + const inline = createInlineLogState(startedAt); + const minLevel = opts.minLevel ?? "debug"; + const severity = { debug: 5, info: 9, warning: 13, error: 17 } as const; + let attributes: RuntimeLogContext = {}; + let explicit: RuntimeOutcome | undefined; + let outcomeMessage: string | undefined; + let error: unknown; + let errorThrown = false; + let sealed = false; + const emitted: ReturnType[] = []; + const base = () => ({ + "autter.operation.id": operationId, + "autter.operation.name": `${method} ${route}`, + "autter.request.id": requestId, + "http.request.method": method, + "http.route": route, + }); + const record = ( + level: RuntimeLogLevel, + message: string, + attrs: RuntimeLogContext, + extra: Record = {}, + ) => { + const user = mergeContext(attributes, attrs, redact); + const exceptions = Object.fromEntries( + Object.entries(user).filter(([key]) => key.startsWith("exception.")), + ); + emitted.push( + createRuntimeEvent({ + level, + message: String(boundContext({ message }, redact).message ?? "").slice(0, 4000), + attributes: boundContext( + { + "autter.event.id": randomId(), + ...extra, + ...exceptions, + ...base(), + ...user, + } as RuntimeLogContext, + redact, + ), + }), + ); + }; + const log = (level: RuntimeLogLevel, message: string, attrs: RuntimeLogContext = {}, extra = {}) => { + if (sealed || severity[level] < severity[minLevel]) return; + const { "exception.stacktrace": _stack, ...small } = userContext(attrs, redact); + appendInlineLog(inline, level, message, small, redact); + if (level === "warning" || level === "error") record(level, message, attrs, extra); + }; + const noteError = (err: unknown, thrownOut = false) => { + if (sealed) return; + error = err; + if (thrownOut) errorThrown = true; + }; + const captured = new WeakSet(); + const rt: EdgeRuntime = { + set(context) { + if (!sealed) attributes = mergeContext(attributes, context, redact); + }, + outcome(status, message) { + if (sealed) return; + explicit = status; + outcomeMessage = message; + }, + info: (message, attrs) => log("info", message, attrs), + warn: (message, attrs) => log("warning", message, attrs), + error(err, attrs) { + noteError(err); + log( + "error", + err instanceof Error ? err.message : String(err), + { + ...attrs, + ...(err instanceof Error + ? { "exception.type": err.name, "exception.stacktrace": err.stack ?? "" } + : {}), + }, + errorAttributes(err), + ); + }, + requestId, + captureException(err, attrs) { + if (sealed) return; + if (err && typeof err === "object") { + if (captured.has(err)) return; + captured.add(err); + } + noteError(err); + const isError = err instanceof Error; + const message = isError ? err.message : String(err); + appendInlineLog(inline, "error", message, undefined, redact); + record( + "error", + message, + { + ...attrs, + "exception.type": isError ? err.name : "Error", + "exception.message": message, + "exception.stacktrace": (isError && err.stack) || new Error(message).stack || "", + }, + { "autter.capture.mode": "log", ...errorAttributes(err) }, + ); + }, + }; + const finish = (status: number, aborted: boolean) => { + if (sealed) return emitted; + const outcome = requestOutcome({ + ...(explicit ? { explicit } : {}), + ...(error !== undefined ? { error } : {}), + ...(errorThrown ? { thrown: true } : {}), + status, + aborted, + }); + const level: RuntimeLogLevel = outcome === "failed" ? "error" : "info"; + record( + level, + outcomeMessage ?? `${method} ${route}: ${outcome}`, + error instanceof Error + ? { "exception.type": error.name, "exception.stacktrace": error.stack ?? "" } + : {}, + { + "autter.event.type": "operation", + "autter.operation.outcome": outcome, + "autter.operation.duration_ms": Date.now() - startedAt, + "autter.operation.steps": [], + "autter.operation.kind": "request", + "autter.operation.level": maxLevel(inline.level, level), + "http.response.status_code": status, + "autter.request.aborted": aborted, + ...(error !== undefined ? errorAttributes(error) : {}), + ...(inline.logsTruncated ? { "autter.operation.logs_truncated": true } : {}), + ...(inline.logs.length ? { "autter.operation.logs": inline.logs } : {}), + }, + ); + sealed = true; + return emitted; + }; + return { rt, finish, noteError }; +} diff --git a/packages/runtime-edge/test/edge.test.mjs b/packages/runtime-edge/test/edge.test.mjs new file mode 100644 index 0000000..9333827 --- /dev/null +++ b/packages/runtime-edge/test/edge.test.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { withAutter, defineRuntimeErrors, RuntimeError, toClientError } from "../dist/index.js"; + +const billing = defineRuntimeErrors("billing", { + declined: { status: 402, message: "Payment declined", expected: true, why: "Issuer rejected the charge" }, +}); + +function harness() { + const posts = []; + const fakeFetch = async (url, init) => { + posts.push({ url, headers: init.headers, body: JSON.parse(init.body) }); + return new Response("{}", { status: 200 }); + }; + const pending = []; + const ctx = { waitUntil: (promise) => pending.push(promise) }; + const records = () => + posts.flatMap((p) => p.body.resourceLogs[0].scopeLogs[0].logRecords).map((r) => ({ + level: r.severityText, + message: r.body.stringValue, + attrs: Object.fromEntries( + r.attributes.map((a) => [ + a.key, + a.value.stringValue ?? a.value.boolValue ?? a.value.doubleValue ?? a.value.kvlistValue ?? a.value.arrayValue, + ]), + ), + })); + return { posts, fakeFetch, ctx, pending, records }; +} + +test("withAutter emits a request summary, echoes the request id and flushes via waitUntil", async () => { + const h = harness(); + const worker = withAutter( + (env) => ({ apiKey: env.AUTTER_RUNTIME_KEY, service: "edge-api", environment: "test", fetch: h.fakeFetch, endpoint: "https://ingest.test/" }), + async (request, env, _ctx, rt) => { + rt.set({ tenant: env.TENANT, password: "hunter2" }); + rt.info("routing", { region: "eu" }); + rt.warn("slow upstream", { ms: 900 }); + return new Response("ok", { headers: { "access-control-allow-origin": "*" } }); + }, + ); + assert.equal(typeof worker.fetch, "function"); + const response = await worker.fetch( + new Request("https://edge.test/orders/123?x=1", { headers: { "x-request-id": "edge-req-0001" } }), + { AUTTER_RUNTIME_KEY: "server-key", TENANT: "acme" }, + h.ctx, + ); + assert.equal(response.headers.get("x-request-id"), "edge-req-0001"); + assert.equal(response.headers.get("access-control-expose-headers"), "x-request-id"); + assert.equal(h.pending.length, 1); + await Promise.all(h.pending); + assert.equal(h.posts[0].url, "https://ingest.test/v1/logs"); + assert.equal(h.posts[0].headers.authorization, "Bearer server-key"); + const records = h.records(); + const summary = records.find((r) => r.attrs["autter.event.type"] === "operation"); + assert.equal(summary.attrs["autter.operation.kind"], "request"); + assert.equal(summary.attrs["autter.operation.name"], "GET /orders/:id"); + assert.equal(summary.attrs["http.route"], "/orders/:id"); + assert.equal(summary.attrs["http.response.status_code"], 200); + assert.equal(summary.attrs["autter.operation.outcome"], "succeeded"); + assert.equal(summary.attrs["autter.request.id"], "edge-req-0001"); + assert.equal(summary.attrs.tenant, "acme"); + assert.equal(summary.attrs.password, "[redacted]"); + assert.equal(summary.attrs["autter.operation.level"], "warning"); + // warn is folded and emitted; info only folded + assert.equal(records.filter((r) => r.attrs["autter.event.type"] !== "operation").length, 1); + assert.equal(records.find((r) => r.level === "WARNING").message, "slow upstream"); + const resource = Object.fromEntries(h.posts[0].body.resourceLogs[0].resource.attributes.map((a) => [a.key, a.value.stringValue])); + assert.deepEqual(resource, { "service.name": "edge-api", "deployment.environment.name": "test" }); +}); + +test("thrown coded errors become promoted error records; expected ones degrade the request", async () => { + const h = harness(); + const worker = withAutter( + { apiKey: "k", service: "edge-api", fetch: h.fakeFetch, errorResponse: true }, + async (request) => { + if (request.url.endsWith("/decline")) throw billing.declined({ internal: { bin: "424242" } }); + throw new Error("lookup for person@example.com exploded"); + }, + ); + const declined = await worker(new Request("https://edge.test/decline", { method: "POST" }), {}, h.ctx); + assert.equal(declined.status, 402); + const body = await declined.json(); + assert.equal(body.error.code, "billing.declined"); + assert.equal(body.error.requestId, declined.headers.get("x-request-id")); + const crashed = await worker(new Request("https://edge.test/crash"), {}, h.ctx); + assert.equal(crashed.status, 500); + assert.deepEqual((await crashed.json()).error.message, "Internal Server Error"); + await Promise.all(h.pending); + const records = h.records(); + const promoted = records.filter((r) => r.attrs["autter.capture.mode"] === "log"); + assert.equal(promoted.length, 2); + const coded = promoted.find((r) => r.attrs["autter.error.code"] === "billing.declined"); + assert.equal(coded.level, "ERROR"); + assert.equal(coded.attrs["exception.type"], "RuntimeError"); + assert.equal(coded.attrs["autter.error.expected"], true); + assert.equal(coded.attrs["http.route"], "/decline"); + const summaries = records.filter((r) => r.attrs["autter.event.type"] === "operation"); + assert.deepEqual(summaries.map((s) => s.attrs["autter.operation.outcome"]), ["degraded", "failed"]); + assert.equal(JSON.stringify(h.posts).includes("424242"), false, "internal never exported"); + assert.equal(JSON.stringify(h.posts).includes("person@example.com"), false, "messages are redacted"); +}); + +test("without errorResponse the error is rethrown after recording; ignore and no-key paths", async () => { + const h = harness(); + const worker = withAutter({ service: "edge-api", fetch: h.fakeFetch, ignore: ["/healthz"] }, async (request) => { + if (request.url.endsWith("/healthz")) return new Response("ok"); + throw new RuntimeError({ code: "edge.bad", message: "bad" }); + }); + const warn = console.warn; + console.warn = () => {}; + try { + await assert.rejects(worker(new Request("https://edge.test/x"), {}, h.ctx), /bad/); + const health = await worker(new Request("https://edge.test/healthz"), {}, h.ctx); + assert.equal(health.headers.get("x-request-id"), null); + await Promise.all(h.pending); + assert.equal(h.posts.length, 0, "no apiKey → nothing exported"); + } finally { + console.warn = warn; + } + assert.deepEqual(toClientError(new RuntimeError({ code: "edge.bad", message: "bad" })), { error: { message: "bad", code: "edge.bad" } }); +}); + +test("the queue is bounded per isolate", async () => { + let calls = 0; + const worker = withAutter( + { apiKey: "k", service: "edge-api", maxQueue: 3, fetch: async () => (calls++, new Response("", { status: 503 })) }, + async (_req, _env, _ctx, rt) => { + for (let i = 0; i < 10; i++) rt.warn(`w${i}`); + return new Response("ok"); + }, + ); + const warn = console.warn; + console.warn = () => {}; + try { + const pending = []; + await worker(new Request("https://edge.test/"), {}, { waitUntil: (p) => pending.push(p) }); + await Promise.all(pending); + assert.equal(calls, 2, "one batch of ≤3 records, retried once on 5xx"); + } finally { + console.warn = warn; + } +}); diff --git a/packages/runtime-edge/tsconfig.json b/packages/runtime-edge/tsconfig.json new file mode 100644 index 0000000..46e9a0f --- /dev/null +++ b/packages/runtime-edge/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"] +} diff --git a/packages/runtime-edge/tsup.config.ts b/packages/runtime-edge/tsup.config.ts new file mode 100644 index 0000000..43c5f7b --- /dev/null +++ b/packages/runtime-edge/tsup.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { index: "src/index.ts" }, + format: ["esm", "cjs"], + dts: { resolve: ["@autter/runtime-core"] }, + // Workers / Vercel Edge / Deno / Bun all run ES2022. + target: "es2022", + platform: "neutral", + clean: true, + // Private workspace package — inlined so the published package keeps + // zero dependencies. + noExternal: ["@autter/runtime-core"], +}); diff --git a/packages/runtime-next/README.md b/packages/runtime-next/README.md index 8cec646..2b84d9d 100644 --- a/packages/runtime-next/README.md +++ b/packages/runtime-next/README.md @@ -103,3 +103,51 @@ See the [operation logging guide](../../docs/OPERATION-LOGGING.md) for measured steps, explicit business outcomes, privacy and flushing. Ordinary error logs are diagnostics; captured exceptions and declared failed outcomes use tracing for issue grouping. + +## Requests, coded errors and edge middleware (1.5.0+) + +Route handlers become request summaries (always kept, request id echoed in +`x-request-id`); the log flush is handed to Next's `after()` automatically +(Next 15+, `unstable_after` on 14.2): + +```ts +// app/api/checkout/route.ts +import { withRuntimeRequest, runtimeContext, defineRuntimeErrors, toClientError } from "@autter/runtime-next"; + +const billing = defineRuntimeErrors("billing", { + declined: { status: 402, message: "Payment declined", expected: true }, +}); + +export const POST = withRuntimeRequest(async (request) => { + runtimeContext.set({ plan: "pro" }); + const ok = await charge(await request.json()); + if (!ok) { + const error = billing.declined(); + return Response.json(toClientError(error, runtimeContext.requestId), { status: 402 }); + } + return Response.json({ ok: true }); +}, { name: "checkout" }); +``` + +Everything new in `@autter/runtime-node` 1.5.0 (`runtimeContext`, +`RuntimeError`, `defineRuntimeErrors`, `runInBackground`, `initAutterLogging`, +enrichers, sinks, …) is re-exported from `@autter/runtime-next` and +`/server`. `@autter/runtime-next/client` adds `autterErrorFromResponse`. + +`middleware.ts` runs on the edge runtime — use `@autter/runtime-next/edge` +(re-exports [`@autter/runtime-edge`](../runtime-edge)): + +```ts +import { NextResponse } from "next/server"; +import { withAutter } from "@autter/runtime-next/edge"; + +export default withAutter( + { apiKey: process.env.AUTTER_RUNTIME_KEY, service: "web-middleware" }, + async (request, _event, _ctx, rt) => { + rt.set({ locale: request.headers.get("accept-language")?.slice(0, 2) }); + return NextResponse.next(); + }, +); +``` + +See [Requests, coded errors and background work](../../docs/REQUESTS-AND-ERRORS.md). diff --git a/packages/runtime-next/package.json b/packages/runtime-next/package.json index 67205e9..54ff5d7 100644 --- a/packages/runtime-next/package.json +++ b/packages/runtime-next/package.json @@ -1,7 +1,7 @@ { "name": "@autter/runtime-next", - "version": "1.4.0", - "description": "One-command Autter Runtime for Next.js: server OTel, browser tracker, relay route, and React error boundary", + "version": "1.5.0", + "description": "One-command Autter Runtime for Next.js: server OTel, request wide events, coded errors, browser tracker, relay route, edge middleware and React error boundary", "license": "MIT", "type": "module", "main": "./dist/index.cjs", @@ -22,6 +22,11 @@ "types": "./dist/client.d.ts", "import": "./dist/client.js", "require": "./dist/client.cjs" + }, + "./edge": { + "types": "./dist/edge.d.ts", + "import": "./dist/edge.js", + "require": "./dist/edge.cjs" } }, "files": [ @@ -33,11 +38,13 @@ "directory": "packages/runtime-next" }, "scripts": { - "build": "tsup" + "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsup", + "test": "npm run build && node --test test/*.test.mjs" }, "dependencies": { - "@autter/runtime-browser": "^1.3.3", - "@autter/runtime-node": "^1.4.0" + "@autter/runtime-browser": "^1.4.0", + "@autter/runtime-edge": "^1.0.0", + "@autter/runtime-node": "^1.5.0" }, "peerDependencies": { "react": ">=18" diff --git a/packages/runtime-next/src/client.ts b/packages/runtime-next/src/client.ts index c6bfa5d..890a59c 100644 --- a/packages/runtime-next/src/client.ts +++ b/packages/runtime-next/src/client.ts @@ -18,6 +18,7 @@ export { setUser, setContext, flush, + autterErrorFromResponse, type AutterSeverity, } from "@autter/runtime-browser"; export { diff --git a/packages/runtime-next/src/edge.ts b/packages/runtime-next/src/edge.ts new file mode 100644 index 0000000..7357dd0 --- /dev/null +++ b/packages/runtime-next/src/edge.ts @@ -0,0 +1,14 @@ +/** + * Edge half of @autter/runtime-next — for `middleware.ts` and + * `export const runtime = "edge"` route handlers. Re-exports + * @autter/runtime-edge (zero dependencies, no Node APIs): + * + * import { withAutter } from "@autter/runtime-next/edge"; + * export default withAutter( + * { apiKey: process.env.AUTTER_RUNTIME_KEY, service: "web-middleware" }, + * async (request, event, _ctx, rt) => NextResponse.next(), + * ); + * + * `event.waitUntil` (NextFetchEvent) is used for delivery automatically. + */ +export * from "@autter/runtime-edge"; diff --git a/packages/runtime-next/src/server.ts b/packages/runtime-next/src/server.ts index 4f8955a..e74dfcf 100644 --- a/packages/runtime-next/src/server.ts +++ b/packages/runtime-next/src/server.ts @@ -22,15 +22,28 @@ * apiKey: process.env.AUTTER_RUNTIME_KEY!, * }); * - * Browser tracker + error boundary live in `@autter/runtime-next/client`. + * 3. Route handlers as request wide events (1.5.0): + * + * import { withRuntimeRequest, runtimeContext } from "@autter/runtime-next"; + * export const POST = withRuntimeRequest(async (request) => { + * runtimeContext.set({ cart: { items: 3 } }); + * return Response.json({ ok: true }); + * }, { name: "checkout" }); + * + * The log flush is handed to Next's `after()` automatically. + * + * Browser tracker + error boundary live in `@autter/runtime-next/client`; + * `middleware.ts` (edge runtime) uses `@autter/runtime-next/edge`. */ import { createBrowserRelayFetchHandler, initAutterServer, + withRuntimeRequest as nodeWithRuntimeRequest, type AutterServer, type AutterServerOptions, type RelayOptions, + type RuntimeRequestOptions, } from "@autter/runtime-node"; export { @@ -56,6 +69,26 @@ export { makeSafeCapture, installAutterAutoFlush, redactAttributes, + // 1.5.0 — request wide events, coded errors, logger-only mode + runtimeContext, + runInBackground, + autterRequests, + autterFastify, + autterErrorResponse, + RuntimeError, + defineRuntimeErrors, + isRuntimeErrorLike, + toClientError, + errorAttributes, + CODE_PATTERN, + initAutterLogging, + enrichUserAgent, + enrichRequestSize, + enrichEdgeGeo, + enrichDeployment, + otlpSink, + consoleSink, + fileSink, } from "@autter/runtime-node"; export type { @@ -69,6 +102,31 @@ export type { AutoFlushOptions, FlushTarget, RedactOptions, + RuntimeOperationKind, + RuntimeOperationOptions, + RuntimeContextHandle, + RuntimeEnricher, + RuntimeEnrichEvent, + RuntimeEnrichContext, + RuntimeEvent, + RuntimeRequestOptions, + AutterRequestsOptions, + AutterErrorResponseOptions, + AutterLogging, + AutterLoggingOptions, + RuntimeErrorDefinition, + RuntimeErrorExtras, + RuntimeErrorOptions, + RuntimeErrorLike, + RuntimeErrorFactory, + RuntimeErrorCatalog, + ClientErrorBody, + RuntimeCarrier, + RuntimeSink, + RuntimeSinkContext, + OtlpSinkOptions, + ConsoleSinkOptions, + FileSinkOptions, } from "@autter/runtime-node"; export type { AutterServer, AutterServerOptions, RelayOptions }; @@ -83,3 +141,50 @@ export function createAutterRelayRoute(options: RelayOptions): { } { return { POST: createBrowserRelayFetchHandler(options) }; } + +type AfterFn = (task: Promise | (() => unknown)) => void; +let after: AfterFn | null | undefined; +// Resolve Next's after() once, eagerly, so it can be called synchronously +// inside the request scope. Absent (Next < 15, or outside Next): fail soft — +// the flush still runs, just without being awaited by the platform. +const afterReady: Promise = (async () => { + try { + const specifier = "next/server"; + const mod = (await import(/* webpackIgnore: true */ specifier)) as { + after?: AfterFn; + unstable_after?: AfterFn; + }; + after = mod.after ?? mod.unstable_after ?? null; + } catch { + after = null; + } + return after; +})(); + +function scheduleAfter(promise: Promise): void { + const run = (fn: AfterFn | null | undefined) => { + if (!fn) return; + try { + fn(promise); + } catch { + /* outside a request scope — the flush is already in flight */ + } + }; + if (after !== undefined) run(after); + else void afterReady.then(run); +} + +/** + * Next.js-aware `withRuntimeRequest`: identical to the runtime-node wrapper, + * but the log flush is handed to `after()` from `next/server` (Next 15+; + * `unstable_after` on 14.2) unless you pass your own `waitUntil`. + */ +export function withRuntimeRequest( + handler: (request: Request, ...rest: A) => Response | Promise, + options: RuntimeRequestOptions = {}, +): (request: Request, ...rest: A) => Promise { + return nodeWithRuntimeRequest(handler, { + ...options, + waitUntil: options.waitUntil ?? scheduleAfter, + }); +} diff --git a/packages/runtime-next/test/next.test.mjs b/packages/runtime-next/test/next.test.mjs new file mode 100644 index 0000000..afb1070 --- /dev/null +++ b/packages/runtime-next/test/next.test.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import * as root from "../dist/index.js"; +import * as server from "../dist/server.js"; +import * as edge from "../dist/edge.js"; + +test("server entry re-exports the 1.5.0 runtime-node APIs (root too)", () => { + for (const name of [ + "withRuntimeRequest", + "runtimeContext", + "runInBackground", + "autterRequests", + "autterErrorResponse", + "RuntimeError", + "defineRuntimeErrors", + "isRuntimeErrorLike", + "toClientError", + "initAutterLogging", + "enrichUserAgent", + "otlpSink", + "consoleSink", + "fileSink", + "registerAutter", + "createAutterRelayRoute", + "withRuntimeOperation", + ]) { + assert.equal(typeof server[name] === "function" || typeof server[name] === "object", true, name); + assert.equal(root[name], server[name], name); + } +}); + +test("edge entry re-exports @autter/runtime-edge", () => { + assert.equal(typeof edge.withAutter, "function"); + assert.equal(typeof edge.defineRuntimeErrors, "function"); + assert.equal(typeof edge.toClientError, "function"); +}); + +test("Next withRuntimeRequest works without next installed (after() fails soft)", async () => { + const handler = server.withRuntimeRequest( + async () => { + server.runtimeContext.set({ ok: true }); + return new Response("ok"); + }, + { name: "ping" }, + ); + const original = console.log; + console.log = () => {}; + try { + const response = await handler(new Request("https://app.test/api/ping", { headers: { "x-request-id": "next-req-0001" } })); + assert.equal(response.headers.get("x-request-id"), "next-req-0001"); + assert.equal(await response.text(), "ok"); + } finally { + console.log = original; + } + // an explicit waitUntil still wins + const waited = []; + const explicit = server.withRuntimeRequest(async () => new Response("ok"), { waitUntil: (p) => waited.push(p) }); + console.log = () => {}; + try { + await explicit(new Request("https://app.test/x")); + } finally { + console.log = original; + } + assert.equal(waited.length, 1); +}); diff --git a/packages/runtime-next/tsup.config.ts b/packages/runtime-next/tsup.config.ts index 92a0731..45618aa 100644 --- a/packages/runtime-next/tsup.config.ts +++ b/packages/runtime-next/tsup.config.ts @@ -6,7 +6,18 @@ export default defineConfig([ format: ["esm", "cjs"], dts: true, target: "node20", - clean: true, + // No `clean` here: the three configs build in parallel and a clean in + // one would race the others' declaration output. The build script + // empties dist/ first instead. + // Optional, resolved at runtime (after() wiring) — never bundled. + external: ["next", "next/server"], + }, + { + entry: { edge: "src/edge.ts" }, + format: ["esm", "cjs"], + dts: true, + target: "es2022", + platform: "neutral", }, { entry: { client: "src/client.ts" }, diff --git a/packages/runtime-node/README.md b/packages/runtime-node/README.md index 1403b01..ac6b30c 100644 --- a/packages/runtime-node/README.md +++ b/packages/runtime-node/README.md @@ -5,6 +5,49 @@ application context to logs and completed operations. See the full [operation logging guide](../../docs/OPERATION-LOGGING.md) for initialization, steps, outcomes, redaction, export lifecycle, and release requirements. +# Requests and coded errors (1.5.0) + +Requires ingester 1.5.0. One summary per request (always kept), request ids +echoed to clients, one issue per error code: + +```ts +import { + autterRequests, autterErrorResponse, defineRuntimeErrors, runtimeContext, +} from "@autter/runtime-node"; + +export const billingErrors = defineRuntimeErrors("billing", { + declined: { status: 402, message: "Payment declined", expected: true, + why: "The card issuer rejected the charge", fix: "Ask for another card" }, + limit: ({ plan }: { plan: string }) => ({ status: 429, message: `Plan ${plan} limit reached` }), +}); + +app.use(autterRequests({ ignore: ["/healthz", "/metrics"] })); +app.post("/checkout", async (req, res) => { + runtimeContext.set({ cart: { items: req.body.items.length } }); + runtimeContext.info("Applied coupon", { coupon: "SPRING" }); // folded into the summary + if (overLimit) throw billingErrors.limit({ plan: "free" }); // Express 5 (next(err) on 4) + res.json({ ok: true, requestId: runtimeContext.requestId }); +}); +app.use(autterErrorResponse()); // { error: { message, code, why?, fix?, link?, requestId } } +``` + +Also: `autterFastify`, `withRuntimeRequest` (fetch-style handlers), +`runtimeContext.fork` / `runInBackground` / `carrier()` for background work and +queues, the per-operation AI usage rollup, `waitUntil`, `initAutterLogging` +(no NodeSDK — for apps with their own OTel), enrichers (`enrichUserAgent`, +`enrichRequestSize`, `enrichEdgeGeo`, `enrichDeployment`), sinks (`otlpSink`, +`consoleSink`, `fileSink` → `.autter/runtime/*.jsonl` in development) and test +helpers: + +```ts +import { captureRuntime, expectOperation } from "@autter/runtime-node/testing"; +const runtime = captureRuntime(); +// … exercise the app … +expectOperation(runtime, "POST /checkout").toHaveOutcome("degraded").toHaveErrorCode("billing.declined"); +``` + +Full guide: [Requests, coded errors and background work](../../docs/REQUESTS-AND-ERRORS.md). + # @autter/runtime-node The server tracker exports per-instance RSS, heap, memory limit, and GC diff --git a/packages/runtime-node/package.json b/packages/runtime-node/package.json index b94a8aa..f3791bb 100644 --- a/packages/runtime-node/package.json +++ b/packages/runtime-node/package.json @@ -1,7 +1,7 @@ { "name": "@autter/runtime-node", - "version": "1.4.0", - "description": "Autter Runtime for Node.js: same-origin browser relay handler + curated OpenTelemetry server tracker", + "version": "1.5.0", + "description": "Autter Runtime for Node.js: request wide events, coded errors, operation logging, same-origin browser relay and a curated OpenTelemetry server tracker", "license": "MIT", "type": "module", "main": "./dist/index.cjs", @@ -12,7 +12,13 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js", "require": "./dist/index.cjs" - } + }, + "./testing": { + "types": "./dist/testing.d.ts", + "import": "./dist/testing.js", + "require": "./dist/testing.cjs" + }, + "./package.json": "./package.json" }, "files": [ "dist" @@ -23,7 +29,7 @@ "directory": "packages/runtime-node" }, "scripts": { - "build": "tsup src/index.ts --format esm,cjs --dts --target node20 --clean", + "build": "tsup", "test": "npm run build && npm run build -w @autter/otlp-ingester && node --test test/*.test.mjs" }, "dependencies": { @@ -39,7 +45,9 @@ "@opentelemetry/semantic-conventions": "^1.28.0" }, "devDependencies": { + "@autter/runtime-core": "0.0.0", "@types/node": "^22.10.0", + "express": "^4.21.2", "tsup": "^8.3.5", "typescript": "^5.7.0" }, diff --git a/packages/runtime-node/src/enrichers.ts b/packages/runtime-node/src/enrichers.ts new file mode 100644 index 0000000..82b5465 --- /dev/null +++ b/packages/runtime-node/src/enrichers.ts @@ -0,0 +1,110 @@ +import { parseUserAgent } from "@autter/runtime-core"; +import type { RuntimeEnricher } from "./logger.js"; + +/** + * Built-in enrichers for `logging.enrich`. They run on operation/request + * summaries before redaction and bounding, and only ever add COARSE values: + * no raw user agents, IPs, cities or headers. + * + * initAutterServer({ ..., logging: { enrich: [enrichUserAgent(), enrichDeployment()] } }); + */ + +function headerOf(req: unknown, name: string): string | undefined { + const headers = (req as { headers?: unknown } | null | undefined)?.headers; + if (!headers) return undefined; + if (typeof (headers as Headers).get === "function") { + return (headers as Headers).get(name) ?? undefined; + } + const value = (headers as Record)[name.toLowerCase()]; + if (Array.isArray(value)) return value[0] === undefined ? undefined : String(value[0]); + return value === undefined || value === null ? undefined : String(value); +} + +/** `user_agent.browser` ("Chrome 120"), `user_agent.os`, `user_agent.device`, `user_agent.bot`. */ +export function enrichUserAgent(): RuntimeEnricher { + return (event, { req }) => { + const ua = headerOf(req, "user-agent"); + if (!ua) return; + const info = parseUserAgent(ua); + if (info.browser) event.attributes["user_agent.browser"] = info.browser; + if (info.os) event.attributes["user_agent.os"] = info.os; + event.attributes["user_agent.device"] = info.device; + event.attributes["user_agent.bot"] = info.bot; + }; +} + +/** `http.request.body.size` / `http.response.body.size` from Content-Length. */ +export function enrichRequestSize(): RuntimeEnricher { + return (event, { req, res }) => { + const request = Number(headerOf(req, "content-length")); + if (Number.isFinite(request) && request >= 0) + event.attributes["http.request.body.size"] = request; + const raw = (res as { getHeader?: (name: string) => unknown } | undefined)?.getHeader?.( + "content-length", + ); + const response = Number(raw); + if (raw !== undefined && Number.isFinite(response) && response >= 0) + event.attributes["http.response.body.size"] = response; + }; +} + +const COUNTRY_HEADERS = [ + "cf-ipcountry", + "x-vercel-ip-country", + "cloudfront-viewer-country", + "x-country-code", + "fastly-geo-country-code", +]; + +/** `geo.country.iso_code` from CDN/edge headers (Cloudflare, Vercel, CloudFront, Fastly). Country only. */ +export function enrichEdgeGeo(): RuntimeEnricher { + return (event, { req }) => { + for (const name of COUNTRY_HEADERS) { + const value = headerOf(req, name)?.trim().toUpperCase(); + if (value && /^[A-Z]{2}$/.test(value) && value !== "XX" && value !== "T1") { + event.attributes["geo.country.iso_code"] = value; + return; + } + } + }; +} + +const REGION_VARS = [ + "AUTTER_REGION", + "AWS_REGION", + "VERCEL_REGION", + "FLY_REGION", + "RAILWAY_REPLICA_REGION", + "GOOGLE_CLOUD_REGION", + "CLOUD_RUN_REGION", + "RENDER_REGION", + "AZURE_REGION", +]; +const COMMIT_VARS = [ + "AUTTER_COMMIT", + "GIT_SHA", + "GIT_COMMIT", + "COMMIT_SHA", + "SOURCE_COMMIT", + "VERCEL_GIT_COMMIT_SHA", + "GITHUB_SHA", + "RENDER_GIT_COMMIT", + "RAILWAY_GIT_COMMIT_SHA", + "CF_PAGES_COMMIT_SHA", + "HEROKU_SLUG_COMMIT", + "SOURCE_VERSION", +]; + +/** `deployment.region` and `deployment.commit` from common platform environment variables. */ +export function enrichDeployment( + env: Record = process.env, +): RuntimeEnricher { + const pick = (names: string[]) => + names.map((name) => env[name]?.trim()).find((value) => value); + const region = pick(REGION_VARS)?.slice(0, 64); + const commit = pick(COMMIT_VARS)?.slice(0, 64); + return (event) => { + if (region) event.attributes["deployment.region"] = region; + if (commit) event.attributes["deployment.commit"] = commit; + }; +} diff --git a/packages/runtime-node/src/errors.ts b/packages/runtime-node/src/errors.ts new file mode 100644 index 0000000..a3a15c3 --- /dev/null +++ b/packages/runtime-node/src/errors.ts @@ -0,0 +1,20 @@ +/** + * Structured, coded errors (shared with @autter/runtime-edge through the + * bundled runtime-core). See docs/REQUESTS-AND-ERRORS.md. + */ +export { + CODE_PATTERN, + RuntimeError, + defineRuntimeErrors, + isRuntimeErrorLike, + toClientError, + errorAttributes, + type RuntimeErrorDefinition, + type RuntimeErrorExtras, + type RuntimeErrorOptions, + type RuntimeErrorLike, + type RuntimeErrorFactory, + type RuntimeErrorCatalog, + type ClientErrorBody, + type RuntimeCarrier, +} from "@autter/runtime-core"; diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index bc3603f..c2fd059 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -44,10 +44,50 @@ export { withRuntimeOperation, flushRuntimeLogs, runtimeLogStats, + runtimeContext, + runInBackground, type RuntimeLogContext, type RuntimeLogContextValue, type RuntimeLogLevel, type RuntimeOutcome, type RuntimeOperation, + type RuntimeOperationKind, + type RuntimeOperationOptions, type RuntimeLoggingOptions, + type RuntimeContextHandle, + type RuntimeEnricher, + type RuntimeEnrichEvent, + type RuntimeEnrichContext, + type RuntimeEvent, } from "./logger.js"; +export { + autterRequests, + autterFastify, + autterErrorResponse, + withRuntimeRequest, + type AutterRequestsOptions, + type AutterErrorResponseOptions, + type RuntimeRequestOptions, +} from "./requests.js"; +export * from "./errors.js"; +export { + initAutterLogging, + type AutterLogging, + type AutterLoggingOptions, +} from "./logging-only.js"; +export { + enrichUserAgent, + enrichRequestSize, + enrichEdgeGeo, + enrichDeployment, +} from "./enrichers.js"; +export { + otlpSink, + consoleSink, + fileSink, + type RuntimeSink, + type RuntimeSinkContext, + type OtlpSinkOptions, + type ConsoleSinkOptions, + type FileSinkOptions, +} from "./sinks.js"; diff --git a/packages/runtime-node/src/llm-instrument.ts b/packages/runtime-node/src/llm-instrument.ts index 849be01..dcec449 100644 --- a/packages/runtime-node/src/llm-instrument.ts +++ b/packages/runtime-node/src/llm-instrument.ts @@ -5,6 +5,7 @@ import { type Span, } from "@opentelemetry/api"; import { autterLlmTracer } from "./server.js"; +import { recordOperationAiUsage } from "./logger.js"; /** * One-line LLM client auto-instrumentation: @@ -69,6 +70,7 @@ function detectProvider(client: object): string { interface UsageShape { inputTokens: number | undefined; outputTokens: number | undefined; + cacheReadTokens?: number | undefined; responseModel: string | undefined; } @@ -83,6 +85,12 @@ function extractUsage(result: unknown): UsageShape { } | null; const usage = r?.usage ?? r?.message?.usage; const meta = r?.usageMetadata; + const details = (usage as { prompt_tokens_details?: { cached_tokens?: unknown } } | undefined) + ?.prompt_tokens_details; + const cached = + usage?.cache_read_input_tokens ?? + (typeof details?.cached_tokens === "number" ? details.cached_tokens : undefined) ?? + meta?.cachedContentTokenCount; const input = usage?.prompt_tokens ?? usage?.input_tokens ?? @@ -97,11 +105,22 @@ function extractUsage(result: unknown): UsageShape { return { inputTokens: typeof input === "number" ? input : undefined, outputTokens: typeof output === "number" ? output : undefined, + cacheReadTokens: typeof cached === "number" ? cached : undefined, responseModel: typeof responseModel === "string" ? responseModel : undefined, }; } +/** Fold one finished call into the current operation's `autter.operation.ai`. */ +function rollUp(model: string, usage: UsageShape): void { + recordOperationAiUsage({ + model: usage.responseModel ?? model, + ...(usage.inputTokens !== undefined ? { inputTokens: usage.inputTokens } : {}), + ...(usage.outputTokens !== undefined ? { outputTokens: usage.outputTokens } : {}), + ...(usage.cacheReadTokens !== undefined ? { cacheReadTokens: usage.cacheReadTokens } : {}), + }); +} + function applyUsage(span: Span, usage: UsageShape): void { if (usage.inputTokens !== undefined) { span.setAttribute("gen_ai.usage.input_tokens", usage.inputTokens); @@ -129,7 +148,7 @@ function endWithError(span: Span, err: unknown): void { * chunk (with `stream_options: { include_usage: true }`), Anthropic's * message_start/message_delta events, Google's per-chunk usageMetadata. */ -function wrapStream(stream: T, span: Span): T { +function wrapStream(stream: T, span: Span, model: string): T { let ended = false; const seen: UsageShape = { inputTokens: undefined, @@ -140,6 +159,7 @@ function wrapStream(stream: T, span: Span): T { if (ended) return; ended = true; applyUsage(span, seen); + rollUp(model, seen); span.setAttribute("autter.llm.streamed", true); if (err !== undefined) { endWithError(span, err); @@ -163,6 +183,7 @@ function wrapStream(stream: T, span: Span): T { const usage = extractUsage(step.value); if (usage.inputTokens !== undefined) seen.inputTokens = usage.inputTokens; if (usage.outputTokens !== undefined) seen.outputTokens = usage.outputTokens; + if (usage.cacheReadTokens !== undefined) seen.cacheReadTokens = usage.cacheReadTokens; if (usage.responseModel) seen.responseModel = usage.responseModel; } else { finish(); @@ -236,9 +257,11 @@ export function instrumentLlmClient( if (!(result instanceof Promise)) { // Anthropic's .stream() returns a MessageStream synchronously. if (result && typeof result === "object" && Symbol.asyncIterator in result) { - return wrapStream(result, span); + return wrapStream(result, span, request.model); } - applyUsage(span, extractUsage(result)); + const usage = extractUsage(result); + applyUsage(span, usage); + rollUp(request.model, usage); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return result; @@ -250,9 +273,11 @@ export function instrumentLlmClient( typeof resolved === "object" && Symbol.asyncIterator in resolved ) { - return wrapStream(resolved, span); + return wrapStream(resolved, span, request.model); } - applyUsage(span, extractUsage(resolved)); + const usage = extractUsage(resolved); + applyUsage(span, usage); + rollUp(request.model, usage); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return resolved; diff --git a/packages/runtime-node/src/logger.ts b/packages/runtime-node/src/logger.ts index 9abf427..b5219b2 100644 --- a/packages/runtime-node/src/logger.ts +++ b/packages/runtime-node/src/logger.ts @@ -1,47 +1,148 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { randomUUID } from "node:crypto"; -import { trace, type Attributes } from "@opentelemetry/api"; +import { + trace, + type Attributes, + type Link, + type SpanContext, +} from "@opentelemetry/api"; +import { + addAiUsage, + appendInlineLog, + boundContext, + createCarrier, + createInlineLogState, + createRuntimeEvent, + errorAttributes, + formatTraceparent, + maxLevel, + mergeContext, + parseCarrier, + parseTraceparent, + toSpanAttributes, + userContext, + LEVEL_SEVERITY, + type AiRollup, + type AiUsageInput, + type InlineLogState, + type RuntimeCarrier, + type RuntimeEvent, + type RuntimeLogContext, + type RuntimeLogLevel, + type RuntimeOperationKind, + type RuntimeOutcome, +} from "@autter/runtime-core"; import { redactAttributes } from "./redact.js"; +import { + consoleSink, + fileSink, + otlpSink, + sinkStats, + type CapturedException, + type FileSinkOptions, + type RuntimeSink, +} from "./sinks.js"; + +export type { + RuntimeLogContext, + RuntimeLogContextValue, + RuntimeLogLevel, + RuntimeOutcome, + RuntimeOperationKind, + RuntimeCarrier, +} from "@autter/runtime-core"; + +/** Mutable view of a record handed to enrichers before redaction/bounding. */ +export interface RuntimeEnrichEvent { + level: RuntimeLogLevel; + message: string; + attributes: Record; +} +export interface RuntimeEnrichContext { + /** The inbound request of a request summary (Node IncomingMessage or fetch Request). */ + req?: unknown; + /** The response object, when the framework exposes one. */ + res?: unknown; + operation: { id: string; name: string; kind: RuntimeOperationKind }; +} +/** Adds attributes to operation/request summaries. Must be cheap; errors are ignored. */ +export type RuntimeEnricher = ( + event: RuntimeEnrichEvent, + context: RuntimeEnrichContext, +) => void; -export type RuntimeLogContextValue = - | string - | number - | boolean - | null - | undefined - | RuntimeLogContextValue[] - | { [key: string]: RuntimeLogContextValue }; -export type RuntimeLogContext = Record; -export type RuntimeLogLevel = "debug" | "info" | "warning" | "error"; -export type RuntimeOutcome = - | "succeeded" - | "failed" - | "degraded" - | "cancelled" - | "pending"; export interface RuntimeLoggingOptions { + /** Print records to the console (default sinks only). Default true. */ console?: boolean; + /** Drop plain log records below this level. Summaries are always kept. */ minLevel?: RuntimeLogLevel; + /** + * Fold debug/info messages logged inside an operation into its summary + * (`autter.operation.logs`, max 50) instead of emitting separate records; + * warn/error are folded AND emitted. Default true. + */ + inline?: boolean; + /** Run on every operation/request summary before redaction. */ + enrich?: RuntimeEnricher[]; + /** + * Replace the default sinks (`[otlpSink(), consoleSink()]`, plus + * `fileSink()` when NODE_ENV=development). + */ + sinks?: RuntimeSink[]; + /** + * Local NDJSON files (`.autter/runtime/YYYY-MM-DD.jsonl`). Default: on only + * when NODE_ENV === "development". Ignored when `sinks` is set. + */ + file?: boolean | FileSinkOptions; + /** + * EXPERIMENTAL zero-code request summaries for plain `node:http` servers + * via the HTTP instrumentation hook (initAutterServer only). Uses + * AsyncLocalStorage.enterWith, which can leak context across keep-alive / + * pipelined requests on one socket — prefer `autterRequests()`. Default false. + */ + requests?: boolean; } + export interface RuntimeOperation { readonly id: string; setContext(attributes: RuntimeLogContext): void; outcome(status: RuntimeOutcome, message?: string): void; step(name: string, fn: () => T | Promise): Promise; } -interface OperationState { + +export interface RuntimeOperationOptions { + /** Carrier from `runtimeContext.carrier()` (object or JSON string): links parent op, request id and trace. */ + from?: RuntimeCarrier | string | null; + /** Receives the log flush promise once the operation completes (serverless `waitUntil`). */ + waitUntil?: (promise: Promise) => void; +} + +/** @internal */ +export interface OperationState { id: string; name: string; + kind: RuntimeOperationKind; parentId?: string; + requestId?: string; + parentTraceId?: string; attributes: RuntimeLogContext; startedAt: number; outcome: RuntimeOutcome; + explicitOutcome?: boolean; message?: string; reportingStack?: string; sealed?: boolean; steps: Array<{ name: string; status: string; durationMs: number }>; + inline: InlineLogState; + ai?: AiRollup; + error?: unknown; + errorThrown?: boolean; + req?: unknown; + res?: unknown; } -interface LoggerConfig { + +/** @internal */ +export interface LoggerConfig { endpoint: string; apiKey: string; service: string; @@ -49,81 +150,62 @@ interface LoggerConfig { release?: string; options?: RuntimeLoggingOptions; redact(attributes: Attributes): Attributes; - run( + /** Wrap an operation in a span; absent in logger-only mode. */ + run?( name: string, fn: () => Promise, attributes: Attributes, + links?: Link[], ): Promise; reportOutcome(name: string, message: string, attributes: Attributes): void; + /** Logger-only mode: record an exception thrown out of an operation. */ + captureThrown?(error: unknown): void; } -const local = new AsyncLocalStorage(); + +/** @internal */ +export const local = new AsyncLocalStorage(); let config: LoggerConfig | null = null; -interface QueuedLog { - record: Record; - bytes: number; -} -let queue: QueuedLog[] = []; -let queueBytes = 0; -let inFlightBytes = 0; -let timer: ReturnType | undefined; -let flushing: Promise | null = null; -let dropped = 0; -let inFlight = 0; -let stopping = false; -let failureStreak = 0; -const severity = { debug: 5, info: 9, warning: 13, error: 17 }; - -function safe(attributes: RuntimeLogContext = {}): RuntimeLogContext { - const redacted = - config?.redact(attributes as Attributes) ?? - redactAttributes(attributes as Attributes); - let budget = 512; - let characters = 16384; - let truncated = false; - const visit = (input: unknown, depth: number): unknown => { - if (--budget < 0 || depth > 6) { - truncated = true; - return "[truncated]"; - } - if (typeof input === "string") { - const text = input.replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1"); - const kept = text.slice(0, Math.max(0, characters)); - truncated ||= kept.length < text.length; - characters -= kept.length; - return kept; - } - if (typeof input === "number") - return Number.isFinite(input) ? input : undefined; - if (typeof input === "boolean" || input === null) return input; - if (Array.isArray(input)) { - truncated ||= input.length > 64; - return input.slice(0, 64).map((item) => visit(item, depth + 1)); - } - if (input && typeof input === "object") { - const entries = Object.entries(input); - if (depth === 0) { - const priority = (key: string) => - /^autter\.(?:operation|event)\./.test(key) - ? 2 - : /^exception\./.test(key) - ? 1 - : 0; - entries.sort(([a], [b]) => priority(b) - priority(a)); - } - truncated ||= entries.length > 100; - return Object.fromEntries( - entries - .slice(0, 100) - .filter(([key]) => !/^(?:__proto__|constructor|prototype)$/.test(key)) - .map(([key, item]) => [key.slice(0, 200), visit(item, depth + 1)]), - ); - } - return undefined; - }; - const result = visit(redacted, 0) as RuntimeLogContext; - if (truncated) result["autter.context.truncated"] = true; - return result; +let sinks: RuntimeSink[] = []; +const extraSinks = new Set(); +let fallbackConsole: RuntimeSink | null = null; + +function redact(attributes: Attributes): Attributes { + return config?.redact(attributes) ?? redactAttributes(attributes); } +const safe = (attributes: RuntimeLogContext = {}) => + boundContext(attributes, redact); +const merge = (current: RuntimeLogContext, next?: RuntimeLogContext) => + mergeContext(current, next, redact); + +function activeSinks(): RuntimeSink[] { + if (config) return extraSinks.size ? [...sinks, ...extraSinks] : sinks; + if (extraSinks.size) return [...extraSinks]; + fallbackConsole ??= consoleSink(); + return [fallbackConsole]; +} + +function defaultSinks(next: LoggerConfig): RuntimeSink[] { + const options = next.options ?? {}; + if (options.sinks) return [...options.sinks]; + const file = + options.file === undefined + ? process.env.NODE_ENV === "development" + : options.file !== false; + return [ + ...(next.apiKey ? [otlpSink()] : []), + ...(options.console !== false ? [consoleSink()] : []), + ...(file + ? [fileSink(typeof options.file === "object" ? options.file : {})] + : []), + ]; +} + +/** @internal The innermost live (unsealed) operation, if any. */ +export function currentOperation(): OperationState | undefined { + const state = local.getStore(); + return state && !state.sealed ? state : undefined; +} + function operationAttributes(state?: OperationState): RuntimeLogContext { return state ? { @@ -132,318 +214,312 @@ function operationAttributes(state?: OperationState): RuntimeLogContext { ...(state.parentId ? { "autter.operation.parent_id": state.parentId } : {}), + ...(state.requestId ? { "autter.request.id": state.requestId } : {}), + ...(state.parentTraceId + ? { "autter.parent_trace_id": state.parentTraceId } + : {}), ...state.attributes, } : {}; } -function userAttributes(attributes: RuntimeLogContext = {}): RuntimeLogContext { - return Object.fromEntries( - Object.entries(safe(attributes)).filter( - ([key]) => !/^autter\.(?:operation|event)\./.test(key), - ), - ); +/** @internal */ +export function traceAttributes(state: OperationState): Attributes { + return toSpanAttributes(operationAttributes(state)); } -function mergeContext( - current: RuntimeLogContext, - next: RuntimeLogContext = {}, -): RuntimeLogContext { - const merge = ( - left: RuntimeLogContext, - right: RuntimeLogContext, - depth: number, - ): RuntimeLogContext => { - const result = { ...left }; - for (const [key, value] of Object.entries(right)) { - const prior = result[key]; - result[key] = - depth < 6 && - value && - typeof value === "object" && - !Array.isArray(value) && - prior && - typeof prior === "object" && - !Array.isArray(prior) - ? merge(prior, value, depth + 1) - : value; - } - return result; - }; - return userAttributes(merge(current, userAttributes(next), 0)); -} -function traceAttributes(state: OperationState): Attributes { - return Object.fromEntries( - Object.entries(operationAttributes(state)) - .filter(([, item]) => item !== null && item !== undefined) - .map(([key, item]) => [ - key, - typeof item === "object" ? JSON.stringify(item) : item, - ]), - ); -} -function scheduleFlush(): void { - if (!timer && config && !stopping && queue.length) { - timer = setTimeout( - () => { - timer = undefined; - void flushRuntimeLogs().catch(() => {}); - }, - Math.min(30000, 2000 * 2 ** Math.min(failureStreak, 4)), - ); - timer.unref(); - } -} -function value(input: unknown): Record { - if (typeof input === "number") return { doubleValue: input }; - if (typeof input === "boolean") return { boolValue: input }; - if (Array.isArray(input)) - return { arrayValue: { values: input.slice(0, 64).map(value) } }; - if (input && typeof input === "object") - return { - kvlistValue: { - values: Object.entries(input) - .slice(0, 128) - .map(([key, item]) => ({ key, value: value(item) })), - }, - }; - return { stringValue: String(input ?? "").slice(0, 32000) }; + +interface EmitOptions { + /** Emit for this operation instead of the ALS one. */ + state?: OperationState | null; + /** Trace context to stamp (summaries finished outside their context). */ + span?: SpanContext; } + function emit( level: RuntimeLogLevel, message: string, attributes: RuntimeLogContext, extra: Record = {}, + options: EmitOptions = {}, ): void { - const state = local.getStore(); + const state = + options.state !== undefined ? (options.state ?? undefined) : local.getStore(); if (state?.sealed) return; // work continuing after completion cannot mutate the emitted operation + const summary = extra["autter.event.type"] === "operation"; if ( - extra["autter.event.type"] !== "operation" && - severity[level] < severity[config?.options?.minLevel ?? "debug"] + !summary && + LEVEL_SEVERITY[level] < LEVEL_SEVERITY[config?.options?.minLevel ?? "debug"] ) return; - const span = trace.getActiveSpan()?.spanContext(); - const user = mergeContext(state?.attributes ?? {}, attributes); + if (!summary && state && config?.options?.inline !== false) { + const { "exception.stacktrace": _stack, ...inlineAttrs } = userContext( + attributes, + redact, + ); + appendInlineLog(state.inline, level, message, inlineAttrs, redact); + if (level === "debug" || level === "info") return; + } + const span = options.span ?? trace.getActiveSpan()?.spanContext(); + const user = merge(state?.attributes ?? {}, attributes); const exceptions = Object.fromEntries( Object.entries(user).filter(([key]) => key.startsWith("exception.")), ); - const attrs = safe({ + const raw: Record = { "autter.event.id": randomUUID(), ...extra, ...exceptions, ...operationAttributes(state), ...user, - } as RuntimeLogContext); - const record = { - timeUnixNano: String(BigInt(Date.now()) * 1_000_000n), - severityNumber: severity[level], - severityText: level.toUpperCase(), - body: value(String(safe({ message }).message ?? "").slice(0, 4000)), - attributes: Object.entries(attrs).map(([key, item]) => ({ - key, - value: value(item), - })), - ...(span && - /^[a-f0-9]{32}$/.test(span.traceId) && - !/^0+$/.test(span.traceId) - ? { traceId: span.traceId, spanId: span.spanId } - : {}), }; - if (config?.options?.console !== false) - console.log( - JSON.stringify({ ...attrs, level, message: record.body.stringValue }), - ); - if (!config) return; - if (stopping) { - dropped++; - return; + const enrichers = config?.options?.enrich; + if (summary && state && enrichers?.length) { + const event: RuntimeEnrichEvent = { level, message, attributes: raw }; + for (const enrich of enrichers) { + try { + enrich(event, { + req: state.req, + res: state.res, + operation: { id: state.id, name: state.name, kind: state.kind }, + }); + } catch { + /* enrichers never break logging */ + } + } } - const bytes = Buffer.byteLength(JSON.stringify(record)); - if ( - bytes > 256 * 1024 || - queue.length + inFlight >= 1000 || - queueBytes + inFlightBytes + bytes > 4 * 1024 * 1024 - ) { - dropped++; - console.warn( - "[autter-runtime] log buffer or record limit reached; record dropped", - ); - return; + const event = createRuntimeEvent({ + level, + message: String(safe({ message }).message ?? "").slice(0, 4000), + attributes: safe(raw as RuntimeLogContext), + ...(span ? { traceId: span.traceId, spanId: span.spanId } : {}), + }); + for (const sink of activeSinks()) { + try { + sink.write(event); + } catch { + /* a broken sink never breaks the application */ + } } - queue.push({ record, bytes }); - queueBytes += bytes; - scheduleFlush(); } +/** @internal */ export function configureRuntimeLogger(next: LoggerConfig): void { config = next; - stopping = false; - failureStreak = 0; + sinks = defaultSinks(next); + for (const sink of sinks) { + try { + sink.start?.({ + endpoint: next.endpoint, + apiKey: next.apiKey, + service: next.service, + environment: next.environment, + ...(next.release ? { release: next.release } : {}), + }); + } catch { + /* ignore */ + } + } +} +/** @internal */ +export function loggerConfigured(): LoggerConfig | null { + return config; } +/** @internal Register a sink for every record regardless of configuration (testing). */ +export function addRuntimeSink(sink: RuntimeSink): () => void { + extraSinks.add(sink); + return () => extraSinks.delete(sink); +} +/** @internal Forward a captured exception to sinks that observe them. */ +export function notifyException(error: unknown, attributes: Record = {}): void { + if (!extraSinks.size && !sinks.some((sink) => sink.exception)) return; + const state = local.getStore(); + const record: CapturedException = { + time: Date.now(), + error, + message: error instanceof Error ? error.message : String(error), + type: error instanceof Error ? error.name : "Error", + attributes: { ...attributes, ...errorAttributes(error) }, + ...(state ? { operationId: state.id } : {}), + ...(state?.requestId ? { requestId: state.requestId } : {}), + }; + for (const sink of [...sinks, ...extraSinks]) { + try { + sink.exception?.(record); + } catch { + /* ignore */ + } + } +} + export function runtimeLogStats(): { buffered: number; dropped: number } { - return { buffered: queue.length + inFlight, dropped }; + return { + buffered: sinks.reduce((sum, sink) => sum + (sink.buffered?.() ?? 0), 0), + dropped: sinkStats.dropped, + }; } export async function flushRuntimeLogs(): Promise { - if (flushing) return flushing; - if (!config || !queue.length) return; - if (timer) { - clearTimeout(timer); - timer = undefined; - } - const activeConfig = config; - const deadline = Date.now() + 10000; - flushing = (async () => { - const count = queue.length; - for (let remaining = count; remaining > 0; ) { - const batch: QueuedLog[] = []; - inFlightBytes = 0; - while (batch.length < Math.min(50, remaining) && queue.length) { - const next = queue[0]!; - if (batch.length && inFlightBytes + next.bytes > 512 * 1024) break; - batch.push(queue.shift()!); - inFlightBytes += next.bytes; - queueBytes -= next.bytes; - } - remaining -= batch.length; - inFlight = batch.length; - const body = JSON.stringify({ - resourceLogs: [ - { - resource: { - attributes: Object.entries({ - "service.name": activeConfig.service, - "deployment.environment.name": activeConfig.environment, - ...(activeConfig.release - ? { "service.version": activeConfig.release } - : {}), - }).map(([key, item]) => ({ key, value: value(item) })), - }, - scopeLogs: [ - { - scope: { name: "autter-runtime" }, - logRecords: batch.map((entry) => entry.record), - }, - ], - }, - ], - }); - let failure: unknown; - for (let attempt = 0; attempt < 3; attempt++) { - try { - if (Date.now() >= deadline) - throw new Error("Runtime log flush exceeded 10 seconds"); - const response = await fetch(`${activeConfig.endpoint}/v1/logs`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${activeConfig.apiKey}`, - }, - body, - signal: AbortSignal.timeout( - Math.max(1, Math.min(3000, deadline - Date.now())), - ), - }); - if (!response.ok) - throw new Error(`Runtime log export failed (${response.status})`); - failure = undefined; - break; - } catch (error) { - failure = error; - } - } - inFlight = 0; - if (failure) { - failureStreak++; - queue = [...batch, ...queue]; - queueBytes += inFlightBytes; - inFlightBytes = 0; - throw failure; - } - inFlightBytes = 0; - failureStreak = 0; - } - })().finally(() => { - flushing = null; - scheduleFlush(); - }); - return flushing; + if (!config) return; + const results = await Promise.allSettled( + sinks.map((sink) => Promise.resolve().then(() => sink.flush?.())), + ); + const failed = results.find((result) => result.status === "rejected"); + if (failed) throw (failed as PromiseRejectedResult).reason; } +/** @internal */ export async function shutdownRuntimeLogger(): Promise { - stopping = true; - try { - await flushRuntimeLogs(); - } finally { - if (queue.length) { - dropped += queue.length; - console.warn( - `[autter-runtime] ${queue.length} log records could not be delivered before shutdown`, - ); - } - queue = []; - queueBytes = 0; - config = null; - if (timer) clearTimeout(timer); - timer = undefined; - } + const active = sinks; + const results = await Promise.allSettled( + active.map((sink) => Promise.resolve().then(() => sink.shutdown?.())), + ); + config = null; + sinks = []; + const failed = results.find((result) => result.status === "rejected"); + if (failed) throw (failed as PromiseRejectedResult).reason; +} + +/** @internal Note an error on the current operation (summary gets autter.error.*). */ +export function noteOperationError( + error: unknown, + thrown = false, + state: OperationState | undefined = currentOperation(), +): void { + if (!state || state.sealed) return; + state.error = error; + if (thrown) state.errorThrown = true; +} + +/** @internal Add one LLM call to the current operation's AI rollup. */ +export function recordOperationAiUsage(usage: AiUsageInput): void { + const state = currentOperation(); + if (state) state.ai = addAiUsage(state.ai, usage); } export function createRuntimeLogger(attributes: RuntimeLogContext = {}) { - const attrs = userAttributes(attributes); + const attrs = userContext(attributes, redact); return { debug: (message: string, extra?: RuntimeLogContext) => - emit("debug", message, mergeContext(attrs, extra)), + emit("debug", message, merge(attrs, extra)), info: (message: string, extra?: RuntimeLogContext) => - emit("info", message, mergeContext(attrs, extra)), + emit("info", message, merge(attrs, extra)), warn: (message: string, extra?: RuntimeLogContext) => - emit("warning", message, mergeContext(attrs, extra)), + emit("warning", message, merge(attrs, extra)), error: (error: unknown, extra?: RuntimeLogContext) => - emit("error", error instanceof Error ? error.message : String(error), { - ...attrs, - ...extra, - ...(error instanceof Error - ? { - "exception.type": error.name, - "exception.stacktrace": error.stack ?? "", - } - : {}), - }), + emit( + "error", + error instanceof Error ? error.message : String(error), + { + ...attrs, + ...extra, + ...(error instanceof Error + ? { + "exception.type": error.name, + "exception.stacktrace": error.stack ?? "", + } + : {}), + }, + errorAttributes(error), + ), }; } export const runtimeLogger = createRuntimeLogger(); -/** Context stays local to this operation; queues/processes require explicit propagation. */ -export function withRuntimeOperation( +/** + * @internal Logger-only/edge-style exception capture: one error record with + * `exception.*`, `autter.error.*` and `autter.capture.mode = "log"`, which + * the ingester promotes to an occurrence. + */ +export function captureExceptionAsLog( + error: unknown, + attributes: Record = {}, +): void { + const isError = error instanceof Error; + const message = isError ? error.message : String(error); + const stack = isError && error.stack + ? error.stack + : (new Error().stack + ?.split("\n") + .filter((line, i) => i === 0 || !/captureException/.test(line)) + .join("\n") ?? ""); + noteOperationError(error); + emit( + "error", + message, + { + ...(attributes as RuntimeLogContext), + "exception.type": isError ? error.name : "Error", + "exception.message": message, + "exception.stacktrace": stack, + }, + { "autter.capture.mode": "log", ...errorAttributes(error) }, + ); +} + +function newState( name: string, - fn: (operation: RuntimeOperation) => T | Promise, - attributes: RuntimeLogContext = {}, -): Promise { - const parent = local.getStore(); - const state: OperationState = { + kind: RuntimeOperationKind, + attributes: RuntimeLogContext, + link: { + parentId?: string; + requestId?: string; + parentTraceId?: string; + base?: RuntimeLogContext; + }, +): OperationState { + const startedAt = Date.now(); + return { id: randomUUID(), name: String(safe({ name }).name).slice(0, 200), - parentId: parent?.sealed ? undefined : parent?.id, - attributes: mergeContext( - parent?.sealed ? {} : (parent?.attributes ?? {}), - attributes, - ), - startedAt: Date.now(), + kind, + ...(link.parentId ? { parentId: link.parentId } : {}), + ...(link.requestId ? { requestId: link.requestId } : {}), + ...(link.parentTraceId ? { parentTraceId: link.parentTraceId } : {}), + attributes: merge(link.base ?? {}, attributes), + startedAt, outcome: "succeeded", steps: [], + inline: createInlineLogState(startedAt), }; - const operation: RuntimeOperation = { +} + +/** @internal Create the state for a request summary (middleware/wrappers). */ +export function createRequestState(input: { + name: string; + requestId: string; + req?: unknown; + res?: unknown; +}): OperationState { + const parent = currentOperation(); + const state = newState(input.name, "request", {}, { + requestId: input.requestId, + ...(parent ? { parentId: parent.id } : {}), + }); + state.req = input.req; + state.res = input.res; + return state; +} + +function setOutcome( + state: OperationState, + status: RuntimeOutcome, + message?: string, +): void { + if (state.sealed) return; + state.outcome = status; + state.explicitOutcome = true; + state.message = message; + if (status === "failed") + state.reportingStack = + new Error("Operation outcome reported here").stack ?? ""; +} + +function operationHandle(state: OperationState): RuntimeOperation { + return { id: state.id, setContext: (attrs) => { if (!state.sealed) { - state.attributes = mergeContext(state.attributes, attrs); + state.attributes = merge(state.attributes, attrs); trace.getActiveSpan()?.setAttributes(traceAttributes(state)); } }, - outcome: (status, message) => { - if (state.sealed) return; - state.outcome = status; - state.message = message; - if (status === "failed") - state.reportingStack = - new Error("Operation outcome reported here").stack ?? ""; - }, + outcome: (status, message) => setOutcome(state, status, message), step: async (stepName, stepFn) => { const started = Date.now(); let status = "succeeded"; @@ -462,6 +538,103 @@ export function withRuntimeOperation( } }, }; +} + +/** @internal Extra summary fields shared by operations and requests. */ +function summaryExtra( + state: OperationState, + level: RuntimeLogLevel, + http: Record = {}, +): Record { + const err = state.error; + return { + "autter.event.type": "operation", + "autter.operation.outcome": state.outcome, + "autter.operation.duration_ms": Date.now() - state.startedAt, + "autter.operation.steps": state.steps, + "autter.operation.kind": state.kind, + "autter.operation.level": maxLevel(state.inline.level, level), + ...(state.ai ? { "autter.operation.ai": state.ai } : {}), + ...http, + ...(err !== undefined ? errorAttributes(err) : {}), + ...(state.inline.logsTruncated + ? { "autter.operation.logs_truncated": true } + : {}), + ...(state.inline.logs.length + ? { "autter.operation.logs": state.inline.logs } + : {}), + }; +} + +/** @internal Emit the summary of an operation or request and seal it. */ +export function emitSummary( + state: OperationState, + options: { http?: Record; span?: SpanContext } = {}, +): void { + if (state.sealed) return; + const err = state.error; + const level: RuntimeLogLevel = state.outcome === "failed" ? "error" : "info"; + emit( + level, + state.message ?? `${state.name}: ${state.outcome}`, + { + ...operationAttributes(state), + ...(err instanceof Error + ? { + "exception.type": err.name, + "exception.stacktrace": err.stack ?? "", + } + : {}), + }, + summaryExtra(state, level, options.http), + { state, ...(options.span ? { span: options.span } : {}) }, + ); + state.sealed = true; +} + +/** @internal Outcome reported via `outcome("failed")` without an exception. */ +export function reportExplicitFailure(state: OperationState): void { + if (state.outcome === "failed" && state.explicitOutcome && !state.errorThrown) + config?.reportOutcome(state.name, state.message ?? "Operation failed", { + ...traceAttributes(state), + "autter.outcome.stack": state.reportingStack ?? "", + }); +} + +interface StartInternal { + /** fork(): link to this operation even when it has already sealed. */ + forkParent?: OperationState; +} + +function startOperation( + name: string, + fn: (operation: RuntimeOperation) => T | Promise, + attributes: RuntimeLogContext, + options: RuntimeOperationOptions, + internal: StartInternal = {}, +): Promise { + const parent = internal.forkParent ?? local.getStore(); + const carrier = options.from ? parseCarrier(options.from) : null; + const traceparent = carrier?.traceparent + ? parseTraceparent(carrier.traceparent) + : null; + const linked = internal.forkParent !== undefined || (parent && !parent.sealed); + const state = newState( + name, + "operation", + attributes, + carrier + ? { + parentId: carrier.op, + ...(carrier.req ? { requestId: carrier.req } : {}), + ...(traceparent ? { parentTraceId: traceparent.traceId } : {}), + } + : { + ...(linked && parent ? { parentId: parent.id, base: parent.attributes } : {}), + ...(parent?.requestId ? { requestId: parent.requestId } : {}), + }, + ); + const operation = operationHandle(state); const run = () => local.run(state, async () => { let thrown: unknown; @@ -470,11 +643,14 @@ export function withRuntimeOperation( } catch (error) { state.outcome = "failed"; thrown = error; + noteOperationError(error, true, state); + config?.captureThrown?.(error); throw error; } finally { trace.getActiveSpan()?.setAttributes({ ...traceAttributes(state), "autter.operation.outcome": state.outcome, + "autter.operation.kind": state.kind, }); if (state.outcome === "failed" && !thrown) config?.reportOutcome( @@ -485,27 +661,125 @@ export function withRuntimeOperation( "autter.outcome.stack": state.reportingStack ?? "", }, ); - emit( - state.outcome === "failed" ? "error" : "info", - state.message ?? `${state.name}: ${state.outcome}`, - { - ...operationAttributes(state), - ...(thrown instanceof Error - ? { - "exception.type": thrown.name, - "exception.stacktrace": thrown.stack ?? "", - } - : {}), - }, - { - "autter.event.type": "operation", - "autter.operation.outcome": state.outcome, - "autter.operation.duration_ms": Date.now() - state.startedAt, - "autter.operation.steps": state.steps, - }, - ); - state.sealed = true; + emitSummary(state); } }); - return config ? config.run(state.name, run, traceAttributes(state)) : run(); + const links: Link[] = traceparent + ? [ + { + context: { + traceId: traceparent.traceId, + spanId: traceparent.spanId, + traceFlags: traceparent.sampled ? 1 : 0, + isRemote: true, + }, + }, + ] + : []; + const promise = config?.run + ? config.run(state.name, run, traceAttributes(state), links) + : run(); + if (!options.waitUntil) return promise; + const waitUntil = options.waitUntil; + return promise.finally(() => { + try { + waitUntil(flushRuntimeLogs().catch(() => {})); + } catch { + /* ignore */ + } + }); } + +/** + * Run `fn` as a named operation: one summary record (outcome, duration, + * steps, context, inline messages) when it ends. Nested operations link via + * `autter.operation.parent_id`; pass `{ from: carrier }` to continue work + * handed over through a queue or another process. + */ +export function withRuntimeOperation( + name: string, + fn: (operation: RuntimeOperation) => T | Promise, + attributes: RuntimeLogContext = {}, + options: RuntimeOperationOptions = {}, +): Promise { + return startOperation(name, fn, attributes, options); +} + +/** + * Like `runtimeContext.fork` but not awaited: errors are recorded on the + * child operation (and its span) and never become unhandled rejections. + */ +export function runInBackground( + name: string, + fn: (operation: RuntimeOperation) => T | Promise, +): Promise { + return runtimeContext.fork(name, fn).catch(() => undefined); +} + +/** + * The current request/operation, from anywhere in its async call tree. + * Every method is a safe no-op (or plain log) outside an operation. + */ +export const runtimeContext = { + /** Merge attributes into the current summary's context. */ + set(attributes: RuntimeLogContext): void { + const state = currentOperation(); + if (!state) return; + state.attributes = merge(state.attributes, attributes); + trace.getActiveSpan()?.setAttributes(traceAttributes(state)); + }, + /** Set the outcome explicitly (wins over status/thrown-error rules). */ + outcome(status: RuntimeOutcome, message?: string): void { + const state = currentOperation(); + if (state) setOutcome(state, status, message); + }, + debug(message: string, attributes?: RuntimeLogContext): void { + runtimeLogger.debug(message, attributes); + }, + info(message: string, attributes?: RuntimeLogContext): void { + runtimeLogger.info(message, attributes); + }, + warn(message: string, attributes?: RuntimeLogContext): void { + runtimeLogger.warn(message, attributes); + }, + /** Log an error and attach it (code/why/fix) to the current summary. */ + error(error: unknown, attributes?: RuntimeLogContext): void { + noteOperationError(error); + runtimeLogger.error(error, attributes); + }, + /** Current operation id. */ + get id(): string | undefined { + return local.getStore()?.id; + }, + /** Current request id (inherited by child operations and forks). */ + get requestId(): string | undefined { + return local.getStore()?.requestId; + }, + /** + * Run `fn` as a child operation whose parent link is captured NOW, so it + * still links after the parent has finished (background work). + */ + fork( + name: string, + fn: (operation: RuntimeOperation) => T | Promise, + ): Promise { + const parent = local.getStore(); + return startOperation(name, fn, {}, {}, parent ? { forkParent: parent } : {}); + }, + /** Serialisable link for queue payloads: `{ v: 1, op, req?, traceparent? }`. */ + carrier(): RuntimeCarrier | undefined { + const state = local.getStore(); + if (!state) return undefined; + const span = trace.getActiveSpan()?.spanContext(); + const traceparent = span + ? formatTraceparent(span.traceId, span.spanId, (span.traceFlags & 1) === 1) + : undefined; + return createCarrier({ + op: state.id, + ...(state.requestId ? { req: state.requestId } : {}), + ...(traceparent ? { traceparent } : {}), + }); + }, +}; +export type RuntimeContextHandle = typeof runtimeContext; +export type { RuntimeEvent }; diff --git a/packages/runtime-node/src/logging-only.ts b/packages/runtime-node/src/logging-only.ts new file mode 100644 index 0000000..ce59886 --- /dev/null +++ b/packages/runtime-node/src/logging-only.ts @@ -0,0 +1,184 @@ +import { + SpanStatusCode, + trace, + type Attributes, + type Span, +} from "@opentelemetry/api"; +import { errorAttributes } from "@autter/runtime-core"; +import { + captureExceptionAsLog, + configureRuntimeLogger, + flushRuntimeLogs, + noteOperationError, + notifyException, + runtimeLogger, + shutdownRuntimeLogger, + type RuntimeLoggingOptions, +} from "./logger.js"; +import { + debugLog, + installAutterAutoFlush, + isDebugEnabled, + registerFlushTarget, + setDebugMode, + unregisterFlushTargets, + type AutoFlushHandle, +} from "./lifecycle.js"; +import { makeRedactor, type RedactOptions } from "./redact.js"; +import { markCaptured } from "./requests.js"; +import { + captureException as serverCaptureException, + serverActive, + setLoggerOnlyCapture, + structuredErrorAttributes, +} from "./server.js"; + +/** + * Logger-only mode: requests, operations, coded errors, sinks and enrichers + * WITHOUT starting the OpenTelemetry NodeSDK. For apps that already run + * their own OTel (a second NodeSDK would fight it) or want logs only. + * + * initAutterLogging({ apiKey: process.env.AUTTER_RUNTIME_KEY, service: "api" }); + * + * Exceptions are recorded on the app's active span when there is one + * (`exceptions: "auto"`); otherwise — or always with `exceptions: "log"` — + * they become error log records with `exception.*` and + * `autter.capture.mode = "log"`, which the ingester (1.5.0+) promotes to + * occurrences. Without an apiKey records go to the console/file sinks only. + */ +export interface AutterLoggingOptions { + /** Private ingest key (autter_rt_…). Omit to log locally only. */ + apiKey?: string; + /** Ingester base URL. Default: https://otlp.autter.dev */ + endpoint?: string; + service: string; + environment?: string; + release?: string; + logging?: RuntimeLoggingOptions; + /** Same semantics as initAutterServer. Default true. */ + redactAttributes?: boolean | RedactOptions; + /** Capture crashes via process.uncaughtExceptionMonitor. Default true. */ + captureGlobalErrors?: boolean; + /** Flush records on beforeExit/SIGINT/SIGTERM. Default true. */ + autoFlush?: boolean; + debug?: boolean; + /** + * "auto" (default): record on the app's active recording span if any, + * else emit a promoted log record. "log": always emit the log record + * (use when the app's own spans are not exported to Autter). + */ + exceptions?: "auto" | "log"; +} + +export interface AutterLogging { + captureException(error: unknown, attributes?: Attributes): void; + flush(): Promise; + shutdown(): Promise; +} + +let activeLogging: AutterLogging | null = null; + +export function initAutterLogging(options: AutterLoggingOptions): AutterLogging { + if (activeLogging) return activeLogging; + if (serverActive()) { + console.warn( + "[autter-runtime] initAutterLogging() ignored: initAutterServer() is already active and includes logging", + ); + return { + captureException: serverCaptureException, + flush: flushRuntimeLogs, + shutdown: async () => {}, + }; + } + if (isDebugEnabled() || options.debug === true) setDebugMode(true); + const endpoint = (options.endpoint ?? "https://otlp.autter.dev").replace(/\/$/, ""); + const environment = options.environment ?? process.env.NODE_ENV ?? "production"; + const redact = makeRedactor(options.redactAttributes ?? true); + const mode = options.exceptions ?? "auto"; + debugLog(`initialising logger-only service=${options.service} endpoint=${endpoint}`); + + function captureException(error: unknown, attributes?: Attributes): void { + markCaptured(error); + const redacted = redact(attributes); + const span: Span | undefined = trace.getActiveSpan(); + if (mode === "auto" && span?.isRecording()) { + noteOperationError(error); + notifyException(error, redacted); + span.setAttributes({ "autter.severity": "error", ...redacted, ...structuredErrorAttributes(error) }); + const declared = errorAttributes(error); + if (error instanceof Error) { + span.addEvent("exception", { + "exception.type": error.name, + "exception.message": error.message, + ...(error.stack ? { "exception.stacktrace": error.stack } : {}), + ...declared, + }); + } else { + span.addEvent("exception", { + "exception.type": "Error", + "exception.message": String(error), + ...declared, + }); + } + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + return; + } + notifyException(error, redacted); + captureExceptionAsLog(error, redacted); + } + + const captured = new WeakSet(); + configureRuntimeLogger({ + endpoint, + apiKey: options.apiKey ?? "", + service: options.service, + environment, + ...(options.release ? { release: options.release } : {}), + ...(options.logging ? { options: options.logging } : {}), + redact, + reportOutcome: (name, message, attributes) => { + runtimeLogger.error(message, { + ...(attributes as Record), + "autter.outcome.name": name.slice(0, 200), + "autter.outcome.message": message.slice(0, 1000), + }); + }, + captureThrown: (error) => { + // One error rethrown through nested operations is one occurrence. + if (error && typeof error === "object") { + if (captured.has(error)) return; + captured.add(error); + } + captureException(error); + }, + }); + setLoggerOnlyCapture(captureException); + + const onCrash = (error: unknown) => + captureException(error, { "autter.unhandled": true }); + if (options.captureGlobalErrors !== false) + process.on("uncaughtExceptionMonitor", onCrash); + + registerFlushTarget("logging", { forceFlush: () => flushRuntimeLogs() }); + let autoFlushHandle: AutoFlushHandle | null = + options.autoFlush === false ? null : installAutterAutoFlush(); + + const handle: AutterLogging = { + captureException, + flush: flushRuntimeLogs, + shutdown: async () => { + process.off("uncaughtExceptionMonitor", onCrash); + setLoggerOnlyCapture(null); + autoFlushHandle?.dispose(); + autoFlushHandle = null; + unregisterFlushTargets(); + activeLogging = null; + await shutdownRuntimeLogger(); + }, + }; + activeLogging = handle; + return handle; +} diff --git a/packages/runtime-node/src/redact.ts b/packages/runtime-node/src/redact.ts index bd93438..6ebcf94 100644 --- a/packages/runtime-node/src/redact.ts +++ b/packages/runtime-node/src/redact.ts @@ -1,408 +1,11 @@ -import type { Attributes } from "@opentelemetry/api"; - /** - * Server-side attribute redaction. The browser relay whitelist-sanitises - * everything a client posts (see relay.ts) — but custom attributes handed to - * `captureException` / `captureMessage` left the process verbatim. This module - * closes that gap: emails, tokens, API keys, and credentials embedded in - * string values are masked, and attributes whose NAME looks sensitive are - * masked wholesale, before anything is exported. + * Server-side attribute redaction. The implementation lives in the private + * @autter/runtime-core package (bundled into this one at build time) so the + * edge SDK masks exactly the same keys and values; this module keeps the + * 1.4.0 import path and public API. */ - -type AttrValue = - | string - | number - | boolean - | Array - | object; - -export interface RedactOptions { - /** - * Extra patterns matched against lower-cased attribute KEYS; a match - * masks the whole value. Strings become case-insensitive substring - * patterns. Extends the built-in list. - */ - additionalKeyPatterns?: (RegExp | string)[]; - /** - * Extra patterns scrubbed inside string VALUES (same semantics as the - * built-in email/token patterns). Extends the built-in list. - */ - additionalValuePatterns?: (RegExp | string)[]; - /** Replacement token. Default "[redacted]". */ - mask?: string; - /** - * Scrub email-shaped substrings from ALL string values, not just - * sensitive keys. Default true. - */ - scrubEmailValues?: boolean; -} - -// Tested against the lower-cased KEY. Deliberately anchored where a loose -// substring would over-redact ("card" must not eat "discard", -// "author" must not eat "author_id"). -const SENSITIVE_KEY_PATTERNS: RegExp[] = [ - /e-?mail/, - /pass(word|wd|phrase)|^pass$/, - /token/, - /secret/, - /credential/, - /(api|access|secret|private|consumer|client|signing|encryption)-?[_.]?key/, - /^-?x?-?authorization$|^auth(-|_|$)|bearer/, - /cookie/, - /phone|msisdn/, - /\bssn\b|social[-_ ]?security/, - /cvv|cvc|card([-_. ]?(number|num|no))?$/, - /credit[-_.]?card/, - /connection[-_.]?string/, - /recovery[-_.]?code|\botp\b|magic[-_.]?link/, -]; - -// Scrubbed INSIDE string values (matched substrings are replaced, the rest -// of the value survives — useful context like a stack frame stays readable). -const PRIVATE_KEY_BLOCK_RE = - /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g; -const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; -const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; -const OPENAI_KEY_RE = /\bsk-[A-Za-z0-9]{20,}\b/g; -const GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g; -const AWS_KEY_RE = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g; -const SLACK_TOKEN_RE = /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g; -const BEARER_RE = /\bbearer\s+[A-Za-z0-9._~+/=-]{10,}/gi; -// postgres://user:password@host — credentials gone, host kept. -const URL_CREDENTIALS_RE = - /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi; - -interface CompiledRedactor { - keyPatterns: RegExp[]; - valuePatterns: RegExp[]; - urlCredentials: RegExp; - privateKeyBlock: RegExp; - mask: string; - scrubEmailValues: boolean; -} - -function toCaseInsensitive(pattern: RegExp | string): RegExp { - return typeof pattern === "string" - ? new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i") - : new RegExp(pattern.source, pattern.flags.includes("i") ? pattern.flags : `${pattern.flags}i`); -} - -function compile(options?: RedactOptions): CompiledRedactor { - const keyPatterns = [...SENSITIVE_KEY_PATTERNS]; - const valuePatterns: RegExp[] = [ - JWT_RE, - OPENAI_KEY_RE, - GITHUB_TOKEN_RE, - AWS_KEY_RE, - SLACK_TOKEN_RE, - BEARER_RE, - ]; - if (options?.additionalKeyPatterns) { - for (const p of options.additionalKeyPatterns) { - keyPatterns.push(toCaseInsensitive(p)); - } - } - if (options?.additionalValuePatterns) { - for (const p of options.additionalValuePatterns) { - if (!valuePatterns.some((existing) => existing.source === (typeof p === "string" ? p : p.source))) { - valuePatterns.push(typeof p === "string" ? new RegExp(p, "gi") : p); - } - } - } - return { - keyPatterns, - valuePatterns, - urlCredentials: URL_CREDENTIALS_RE, - privateKeyBlock: PRIVATE_KEY_BLOCK_RE, - mask: options?.mask ?? "[redacted]", - scrubEmailValues: options?.scrubEmailValues !== false, - }; -} - -function redactString(value: string, r: CompiledRedactor): string { - let out = value.replace(r.privateKeyBlock, r.mask); - for (const re of r.valuePatterns) { - out = out.replace(re, r.mask); - } - out = out.replace(r.urlCredentials, "$1" + r.mask + "@"); - if (r.scrubEmailValues) { - out = out.replace(EMAIL_VALUE_RE, r.mask); - } - return out; -} - -const MAX_REDACTION_DEPTH = 64; -const MAX_REDACTION_WORK = 10_000; -const MAX_COLLECTION_ENTRIES = 1_000; - -interface RedactionState { - ancestors: WeakMap; - remainingWork: number; -} - -function canTraverse(depth: number, state: RedactionState): boolean { - if (depth > MAX_REDACTION_DEPTH || state.remainingWork <= 0) return false; - state.remainingWork -= 1; - return true; -} - -function redactValue( - value: unknown, - r: CompiledRedactor, - state: RedactionState, - depth: number, -): unknown { - if (typeof value === "string") return redactString(value, r); - - let isArray = false; - try { - isArray = Array.isArray(value); - } catch { - return r.mask; - } - - if (isArray) { - return redactArray(value as unknown[], r, state, depth); - } - - if (typeof value === "object" && value !== null) { - return redactObject(value, r, state, depth); - } - - return value; -} - -function redactArray( - value: unknown[], - r: CompiledRedactor, - state: RedactionState, - depth: number, -): unknown { - const existing = state.ancestors.get(value); - if (existing) return r.mask; - - if (!canTraverse(depth, state)) return r.mask; - - const out: unknown[] = []; - state.ancestors.set(value, out); - - let length: number; - try { - length = value.length; - } catch { - state.ancestors.delete(value); - return r.mask; - } - - const limit = Math.min(length, MAX_COLLECTION_ENTRIES); - - for (let i = 0; i < limit; i += 1) { - let item: unknown; - - try { - item = value[i]; - } catch { - out.push(r.mask); - continue; - } - - out.push(redactValue(item, r, state, depth + 1)); - } - - if (length > limit) { - out.push(r.mask); - } - - state.ancestors.delete(value); - return out; -} - -function redactObject( - value: object, - r: CompiledRedactor, - state: RedactionState, - depth: number, -): unknown { - const existing = state.ancestors.get(value); - if (existing) return r.mask; - - if (!canTraverse(depth, state)) return r.mask; - - const out: Record = {}; - state.ancestors.set(value, out); - - let count = 0; - let truncated = false; - - try { - for (const key in value as Record) { - if ( - !Object.prototype.propertyIsEnumerable.call( - value, - key, - ) - ) { - continue; - } - - if (count >= MAX_COLLECTION_ENTRIES) { - truncated = true; - break; - } - - let nestedValue: unknown; - try { - nestedValue = (value as Record)[key]; - } catch { - out[key] = r.mask; - count += 1; - continue; - } - - count += 1; - out[key] = isSensitiveKey(key, nestedValue, r) - ? r.mask - : redactValue( - nestedValue, - r, - state, - depth + 1, - ); - } - } catch { - truncated = true; - } - - if (truncated) { - out.__redaction_truncated__ = r.mask; - } - - state.ancestors.delete(value); - return out; -} -const USAGE_TOKEN_KEYS = new Set([ - "input_tokens", - "output_tokens", - "prompt_tokens", - "completion_tokens", - "total_tokens", - "token_count", - "gen_ai.usage.input_tokens", - "gen_ai.usage.output_tokens", - "gen_ai.usage.prompt_tokens", - "gen_ai.usage.completion_tokens", - "gen_ai.usage.total_tokens", - "gen_ai.usage.token_count", -]); - -function isSensitiveKey( - key: string, - value: unknown, - r: CompiledRedactor, -): boolean { - const lowered = key.toLowerCase(); - - // Canonical GenAI usage attributes are safe when they contain - // valid non-negative numeric counts. - if (USAGE_TOKEN_KEYS.has(lowered)) { - return !( - typeof value === "number" && - Number.isFinite(value) && - value >= 0 - ); - } - - // All other sensitive keys, including token-like keys, are redacted. - return r.keyPatterns.some((re) => re.test(lowered)); -} - -/** - * Return a copy of `attributes` with PII/secrets masked. Never mutates the - * input; non-string primitives pass through untouched; `undefined` values - * are dropped (OpenTelemetry rejects them). - */ -export function redactAttributes( - attributes?: Attributes | null, - options?: RedactOptions, -): Attributes { - const r = compile(options); - return redactWith(attributes, r); -} - -function redactWith( - attributes: Attributes | null | undefined, - r: CompiledRedactor, -): Attributes { - const out: Attributes = {}; - if (!attributes) return out; - - const state: RedactionState = { - ancestors: new WeakMap(), - remainingWork: MAX_REDACTION_WORK, - }; - - let count = 0; - let truncated = false; - - try { - for (const key in attributes as Record) { - if ( - !Object.prototype.propertyIsEnumerable.call( - attributes, - key, - ) - ) { - continue; - } - - if ( - count >= MAX_COLLECTION_ENTRIES || - state.remainingWork <= 0 - ) { - truncated = true; - break; - } - - let value: unknown; - try { - value = (attributes as Record)[key]; - } catch { - out[key] = r.mask; - count += 1; - state.remainingWork -= 1; - continue; - } - - count += 1; - state.remainingWork -= 1; - - if (value === undefined) continue; - - out[key] = isSensitiveKey(key, value, r) - ? r.mask - : (redactValue(value, r, state, 0) as Attributes[string]); - } - } catch { - truncated = true; - } - - if (truncated) { - out.__redaction_truncated__ = r.mask; - } - - return out; -} - -/** - * Compile a redactor once for a hot path — initAutterServer builds one from - * its options and reuses it for every capture instead of recompiling. - */ -export function makeRedactor( - options?: boolean | RedactOptions, -): (attributes?: Attributes | null) => Attributes { - if (options === false) { - return (attributes) => ({ ...(attributes ?? {}) }); - } - const r = compile(options === true ? undefined : options); - return (attributes) => redactWith(attributes, r); -} +export { + redactAttributes, + makeRedactor, + type RedactOptions, +} from "@autter/runtime-core"; diff --git a/packages/runtime-node/src/requests.ts b/packages/runtime-node/src/requests.ts new file mode 100644 index 0000000..64166f6 --- /dev/null +++ b/packages/runtime-node/src/requests.ts @@ -0,0 +1,613 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { AsyncResource } from "node:async_hooks"; +import { context, trace, type SpanContext } from "@opentelemetry/api"; +import { + DEFAULT_REQUEST_ID_HEADER, + RuntimeError, + compileIgnore, + errorCodeOf, + normalizeRoutePath, + requestOutcome, + resolveRequestId, + responseStatusOf, + toClientError, +} from "@autter/runtime-core"; +import { + createRequestState, + emitSummary, + flushRuntimeLogs, + local, + noteOperationError, + reportExplicitFailure, + type OperationState, +} from "./logger.js"; + +/** + * Request wide events: one `kind: "request"` summary per HTTP request with + * method, route template, status, outcome, duration, the request id, the + * context set via `runtimeContext.set()` from anywhere in the request, inline + * messages and child operations (linked by parent id). Always kept — there + * is no sampler; use `ignore` for health checks and metrics endpoints. + */ + +const REQUEST_STATE = Symbol.for("autter.runtime.request"); +const BOUND = Symbol.for("autter.runtime.bound"); +const CAPTURED = new WeakSet(); + +type ExceptionCapturer = (error: unknown) => void; +let capturer: ExceptionCapturer = () => {}; +/** @internal server.ts registers the module-level captureException here. */ +export function setRequestExceptionCapturer(fn: ExceptionCapturer): void { + capturer = fn; +} +/** @internal Remember errors already captured so boundaries don't double-report. */ +export function markCaptured(error: unknown): void { + if (error && typeof error === "object") CAPTURED.add(error); +} + +export interface AutterRequestsOptions { + /** Path globs never summarised: `*` = one segment, `**` = any depth. */ + ignore?: string[]; + /** Request id header to honour and echo. Default "x-request-id". */ + requestIdHeader?: string; + /** Honour an inbound request id that matches `^[\w.-]{8,128}$`. Default true. */ + trustRequestId?: boolean; +} + +export interface AutterErrorResponseOptions { + /** + * Report the error as an exception. Default: errors with status ≥ 500, + * RuntimeErrors and any error with a valid code (expected ones included — + * they are recorded but never open incidents). + */ + capture?: boolean | ((error: unknown) => boolean); + /** Header the request id is read from when no request summary is active. */ + requestIdHeader?: string; +} + +export interface RuntimeRequestOptions { + /** Summary name suffix (default: the route). Name = `METHOD name`. */ + name?: string; + /** Route template for `http.route` (default: pathname with ids → `:id`). */ + route?: string; + requestIdHeader?: string; + trustRequestId?: boolean; + ignore?: string[]; + /** Receives the log flush promise after the response (serverless `waitUntil`). */ + waitUntil?: (promise: Promise) => void; +} + +/** Express (4/5) assigns routing state onto the core request object; a + * matched handler leaves the route template on `req.route.path` and the + * mount prefix on `req.baseUrl`. */ +interface ExpressRequestProps { + baseUrl?: unknown; + route?: { path?: unknown }; +} + +/** + * Route template of a finished Express request ("/api/users/:id"), or null + * when no route matched (404s, static files) or the server isn't Express. + * Only meaningful at response end — Express fills `req.route` during routing. + */ +export function expressRouteOf(req: unknown): string | null { + const props = req as ExpressRequestProps | null | undefined; + const path = props?.route?.path; + if (typeof path !== "string" || path === "") return null; + const base = typeof props?.baseUrl === "string" ? props.baseUrl : ""; + const route = base + path; + return route.startsWith("/") ? route : null; +} + +function defaultShouldCapture(error: unknown): boolean { + return ( + error instanceof RuntimeError || + errorCodeOf(error) !== undefined || + responseStatusOf(error) >= 500 + ); +} + +function captureAtBoundary( + error: unknown, + state: OperationState | undefined, + rule: boolean | ((error: unknown) => boolean) | undefined = undefined, +): void { + const wanted = + typeof rule === "function" + ? rule(error) + : rule === undefined + ? defaultShouldCapture(error) + : rule; + if (!wanted) return; + if (error && typeof error === "object" && CAPTURED.has(error)) return; + markCaptured(error); + try { + if (state && !state.sealed) local.run(state, () => capturer(error)); + else capturer(error); + } catch { + /* capture never breaks the response */ + } +} + +/** + * Re-enter the request's async context in every listener registered on + * `emitter` from now on. Body parsers read the request stream from socket + * callbacks, which would otherwise lose the AsyncLocalStorage store. + */ +function bindEmitter(emitter: unknown, state: OperationState): void { + const target = emitter as Record | null; + if (!target || typeof target !== "object" || target[BOUND]) return; + target[BOUND] = true; + const wrappers = new WeakMap unknown>(); + const wrap = (listener: unknown) => { + if (typeof listener !== "function") return listener; + let wrapped = wrappers.get(listener); + if (!wrapped) { + wrapped = function (this: unknown, ...args: unknown[]) { + return local.run(state, () => + (listener as (...a: unknown[]) => unknown).apply(this, args), + ); + }; + wrappers.set(listener, wrapped); + } + return wrapped; + }; + for (const method of [ + "on", + "addListener", + "prependListener", + "once", + "prependOnceListener", + ]) { + const original = target[method]; + if (typeof original !== "function") continue; + target[method] = function (this: unknown, event: unknown, listener: unknown) { + return (original as (...a: unknown[]) => unknown).call(this, event, wrap(listener)); + }; + } + for (const method of ["removeListener", "off"]) { + const original = target[method]; + if (typeof original !== "function") continue; + target[method] = function (this: unknown, event: unknown, listener: unknown) { + const wrapped = + typeof listener === "function" ? (wrappers.get(listener) ?? listener) : listener; + return (original as (...a: unknown[]) => unknown).call(this, event, wrapped); + }; + } +} + +/** Append the request id header to Access-Control-Expose-Headers when the + * response carries CORS headers, so cross-origin browsers can read it. */ +function exposeHeaderValue(current: unknown, header: string): string | null { + const text = Array.isArray(current) ? current.join(", ") : String(current ?? ""); + const names = text + .split(",") + .map((item) => item.trim().toLowerCase()) + .filter(Boolean); + if (names.includes("*") || names.includes(header)) return null; + return names.length ? `${text}, ${header}` : header; +} + +function exposeOnCors(res: ServerResponse, header: string): void { + const original = res.writeHead; + res.writeHead = function (this: ServerResponse, ...args: unknown[]) { + try { + const inline = args + .slice(1) + .find( + (arg): arg is Record => + !!arg && typeof arg === "object" && !Array.isArray(arg), + ); + const inlineKey = (name: string) => + inline ? Object.keys(inline).find((key) => key.toLowerCase() === name) : undefined; + const cors = + res.getHeader("access-control-allow-origin") !== undefined || + inlineKey("access-control-allow-origin") !== undefined; + if (cors) { + const key = inlineKey("access-control-expose-headers"); + const value = exposeHeaderValue( + key && inline ? inline[key] : res.getHeader("access-control-expose-headers"), + header, + ); + if (value && key && inline) inline[key] = value; + else if (value) res.setHeader("access-control-expose-headers", value); + } + } catch { + /* never block the response */ + } + return (original as (...a: unknown[]) => ServerResponse).apply(this, args); + } as ServerResponse["writeHead"]; +} + +function pathOf(req: IncomingMessage): string { + const raw = + (req as IncomingMessage & { originalUrl?: string }).originalUrl ?? req.url ?? "/"; + return raw.split(/[?#]/)[0] || "/"; +} + +function finishRequest( + state: OperationState, + input: { + method: string; + route: string; + name?: string; + status: number; + aborted: boolean; + span?: SpanContext; + }, +): void { + if (state.sealed) return; + state.name = (input.name ?? `${input.method} ${input.route}`).slice(0, 200); + state.outcome = requestOutcome({ + ...(state.explicitOutcome ? { explicit: state.outcome } : {}), + ...(state.error !== undefined ? { error: state.error } : {}), + ...(state.errorThrown ? { thrown: true } : {}), + status: input.status, + aborted: input.aborted, + }); + const span = trace.getActiveSpan(); + if (span?.isRecording()) span.setAttribute("autter.operation.outcome", state.outcome); + if (input.span) { + const remote = trace.wrapSpanContext(input.span); + context.with(trace.setSpan(context.active(), remote), () => + reportExplicitFailure(state), + ); + } else reportExplicitFailure(state); + emitSummary(state, { + http: { + "http.request.method": input.method, + "http.route": input.route, + "http.response.status_code": input.status, + "autter.request.aborted": input.aborted, + }, + ...(input.span ? { span: input.span } : {}), + }); +} + +/** Start a request summary for a Node request/response pair. */ +function beginNodeRequest( + req: IncomingMessage, + res: ServerResponse, + options: AutterRequestsOptions, + route?: () => string | null, +): OperationState { + const header = (options.requestIdHeader ?? DEFAULT_REQUEST_ID_HEADER).toLowerCase(); + const requestId = resolveRequestId( + options.trustRequestId === false ? undefined : req.headers[header], + ); + const method = (req.method ?? "GET").toUpperCase(); + const path = pathOf(req); + const state = createRequestState({ + name: `${method} ${normalizeRoutePath(path)}`, + requestId, + req, + res, + }); + (req as unknown as Record)[REQUEST_STATE] = state; + try { + if (!res.headersSent) res.setHeader(header, requestId); + exposeOnCors(res, header); + } catch { + /* headers already sent */ + } + const span = trace.getActiveSpan(); + if (span?.isRecording()) + span.setAttributes({ + "autter.request.id": requestId, + "autter.operation.kind": "request", + }); + const spanContext = span?.spanContext(); + bindEmitter(req, state); + bindEmitter(res, state); + let finished = false; + const done = () => { + finishRequest(state, { + method, + route: route?.() ?? expressRouteOf(req) ?? normalizeRoutePath(path), + status: res.statusCode, + aborted: !finished, + ...(spanContext ? { span: spanContext } : {}), + }); + }; + res.once("finish", () => { + finished = true; + done(); + }); + res.once("close", done); + return state; +} + +/** + * Express / Connect middleware. Mount it early (before routers; after body + * parsers is fine too): + * + * app.use(autterRequests({ ignore: ["/healthz", "/metrics"] })); + * + * Echoes the request id in the response (`x-request-id`), stamps it on the + * server span and every record, and emits the summary on finish/close. + */ +export function autterRequests(options: AutterRequestsOptions = {}) { + const ignored = compileIgnore(options.ignore); + return function autterRequestsMiddleware( + req: IncomingMessage, + res: ServerResponse, + next: (error?: unknown) => void, + ): void { + if ((req as unknown as Record)[REQUEST_STATE] || ignored(pathOf(req))) { + next(); + return; + } + const state = beginNodeRequest(req, res, options); + local.run(state, () => next()); + }; +} + +/** + * Express error middleware: records the error on the request summary, + * reports it (see `capture`), and answers with + * `{ error: { message, code?, why?, fix?, link?, requestId? } }` and the + * error's status (default 500). Never includes `internal`, stacks or causes; + * unstructured 5xx messages are replaced with a generic one. + * + * app.use(autterErrorResponse()); // after your routes + */ +export function autterErrorResponse(options: AutterErrorResponseOptions = {}) { + return function autterErrorResponseMiddleware( + error: unknown, + req: IncomingMessage, + res: ServerResponse, + next: (error?: unknown) => void, + ): void { + const state = (req as unknown as Record)[REQUEST_STATE]; + if (state) noteOperationError(error, true, state); + captureAtBoundary(error, state, options.capture); + if (res.headersSent) { + next(error); + return; + } + const header = (options.requestIdHeader ?? DEFAULT_REQUEST_ID_HEADER).toLowerCase(); + const inbound = req.headers?.[header]; + const requestId = + state?.requestId ?? (typeof inbound === "string" ? inbound.slice(0, 128) : undefined); + res.statusCode = responseStatusOf(error); + res.setHeader("content-type", "application/json; charset=utf-8"); + res.end(JSON.stringify(toClientError(error, requestId))); + }; +} + +// --------------------------------------------------------------------------- +// Fastify (no dependency — typed structurally) +// --------------------------------------------------------------------------- + +interface FastifyLikeRequest { + url: string; + method: string; + raw: IncomingMessage; + headers: Record; + routeOptions?: { url?: string }; + routerPath?: string; + [key: symbol]: unknown; +} +interface FastifyLikeReply { + raw: ServerResponse; + statusCode: number; + header(name: string, value: string): unknown; + getHeader(name: string): unknown; +} +type HookDone = (error?: Error) => void; +interface FastifyLikeInstance { + addHook(name: string, hook: (...args: never[]) => unknown): unknown; +} + +const RESOURCE = Symbol.for("autter.runtime.resource"); + +/** + * Fastify plugin (Fastify 4/5, no dependency on fastify itself): + * + * app.register(autterFastify, { ignore: ["/healthz"] }); + * + * Skips plugin encapsulation, so its hooks cover every route. + */ +export function autterFastify( + fastify: FastifyLikeInstance, + options: AutterRequestsOptions | undefined, + done: (error?: Error) => void, +): void { + const opts = options ?? {}; + const header = (opts.requestIdHeader ?? DEFAULT_REQUEST_ID_HEADER).toLowerCase(); + const ignored = compileIgnore(opts.ignore); + const stateOf = (request: FastifyLikeRequest) => + request[REQUEST_STATE] as OperationState | undefined; + fastify.addHook("onRequest", ((request: FastifyLikeRequest, reply: FastifyLikeReply, next: HookDone) => { + if (ignored(request.url) || request.raw[REQUEST_STATE as never]) { + next(); + return; + } + const requestId = resolveRequestId( + opts.trustRequestId === false ? undefined : request.headers[header], + ); + const method = String(request.method ?? "GET").toUpperCase(); + const state = createRequestState({ + name: `${method} ${normalizeRoutePath(request.url)}`, + requestId, + req: request.raw, + res: reply.raw, + }); + request[REQUEST_STATE] = state; + (request.raw as unknown as Record)[REQUEST_STATE] = state; + reply.header(header, requestId); + const span = trace.getActiveSpan(); + if (span?.isRecording()) + span.setAttributes({ + "autter.request.id": requestId, + "autter.operation.kind": "request", + }); + const spanContext = span?.spanContext(); + bindEmitter(request.raw, state); + bindEmitter(reply.raw, state); + let finished = false; + const finish = (aborted: boolean) => + finishRequest(state, { + method, + route: + request.routeOptions?.url ?? + request.routerPath ?? + normalizeRoutePath(request.url), + status: reply.raw.statusCode || reply.statusCode, + aborted, + ...(spanContext ? { span: spanContext } : {}), + }); + reply.raw.once("finish", () => { + finished = true; + finish(false); + }); + reply.raw.once("close", () => finish(!finished)); + local.run(state, () => { + request[RESOURCE] = new AsyncResource("autter-request"); + next(); + }); + }) as never); + // Body parsing runs from socket callbacks; re-enter before the handler. + fastify.addHook("preHandler", ((request: FastifyLikeRequest, _reply: FastifyLikeReply, next: HookDone) => { + const resource = request[RESOURCE] as AsyncResource | undefined; + if (resource) resource.runInAsyncScope(() => next()); + else next(); + }) as never); + fastify.addHook("onSend", ((request: FastifyLikeRequest, reply: FastifyLikeReply, payload: unknown, next: (error: Error | null, payload?: unknown) => void) => { + try { + if (stateOf(request) && reply.getHeader("access-control-allow-origin") !== undefined) { + const value = exposeHeaderValue(reply.getHeader("access-control-expose-headers"), header); + if (value) reply.header("access-control-expose-headers", value); + } + } catch { + /* ignore */ + } + next(null, payload); + }) as never); + fastify.addHook("onError", ((request: FastifyLikeRequest, _reply: FastifyLikeReply, error: unknown, next: HookDone) => { + const state = stateOf(request); + if (state) noteOperationError(error, true, state); + captureAtBoundary(error, state); + next(); + }) as never); + done(); +} +(autterFastify as unknown as Record)[Symbol.for("skip-override")] = true; +(autterFastify as unknown as Record)[Symbol.for("fastify.display-name")] = + "autter-runtime"; + +// --------------------------------------------------------------------------- +// fetch-style handlers (Next.js route handlers, Hono on Node, Remix, …) +// --------------------------------------------------------------------------- + +function withHeaders(response: Response, header: string, requestId: string): Response { + const apply = (target: Response) => { + target.headers.set(header, requestId); + if (target.headers.get("access-control-allow-origin") !== null) { + const value = exposeHeaderValue( + target.headers.get("access-control-expose-headers") ?? "", + header, + ); + if (value) target.headers.set("access-control-expose-headers", value); + } + return target; + }; + try { + return apply(response); + } catch { + // Immutable headers (e.g. a proxied fetch() response): copy once. + try { + return apply(new Response(response.body, response)); + } catch { + return response; + } + } +} + +/** + * Wrap a fetch-style handler `(request, ...rest) => Response` in a request + * summary. Thrown errors are recorded and reported, then rethrown. + * + * export const POST = withRuntimeRequest(handler, { name: "checkout" }); + */ +export function withRuntimeRequest( + handler: (request: Request, ...rest: A) => Response | Promise, + options: RuntimeRequestOptions = {}, +): (request: Request, ...rest: A) => Promise { + const header = (options.requestIdHeader ?? DEFAULT_REQUEST_ID_HEADER).toLowerCase(); + const ignored = compileIgnore(options.ignore); + return async (request: Request, ...rest: A): Promise => { + let pathname = "/"; + try { + pathname = new URL(request.url, "http://localhost").pathname; + } catch { + /* keep "/" */ + } + if (ignored(pathname)) return handler(request, ...rest); + const requestId = resolveRequestId( + options.trustRequestId === false ? undefined : request.headers.get(header), + ); + const method = String(request.method ?? "GET").toUpperCase(); + const route = options.route ?? normalizeRoutePath(pathname); + const state = createRequestState({ + name: `${method} ${options.name ?? route}`, + requestId, + req: request, + }); + const span = trace.getActiveSpan(); + if (span?.isRecording()) + span.setAttributes({ + "autter.request.id": requestId, + "autter.operation.kind": "request", + }); + const spanContext = span?.spanContext(); + let status = 500; + try { + const response = await local.run(state, async () => { + try { + return await handler(request, ...rest); + } catch (error) { + noteOperationError(error, true, state); + captureAtBoundary(error, state, true); + throw error; + } + }); + status = response.status; + return withHeaders(response, header, requestId); + } finally { + const aborted = request.signal?.aborted === true; + finishRequest(state, { + method, + route, + name: `${method} ${options.name ?? route}`, + status, + aborted, + ...(spanContext ? { span: spanContext } : {}), + }); + if (options.waitUntil) { + try { + options.waitUntil(flushRuntimeLogs().catch(() => {})); + } catch { + /* ignore */ + } + } + } + }; +} + +// --------------------------------------------------------------------------- +// Zero-code hook mode (logging.requests: true) — EXPERIMENTAL +// --------------------------------------------------------------------------- + +/** + * @internal Called from the HTTP instrumentation's response hook for every + * incoming request when `logging.requests` is on. Enters the request store + * with `enterWith`, which can bleed into later work on the same socket + * (keep-alive / pipelining) — that is why the mode is off by default. + */ +export function startHookRequest(res: ServerResponse): void { + const req = (res as ServerResponse & { req?: IncomingMessage }).req; + if (!req || (req as unknown as Record)[REQUEST_STATE]) return; + const state = beginNodeRequest(req, res, {}); + local.enterWith(state); +} diff --git a/packages/runtime-node/src/server.ts b/packages/runtime-node/src/server.ts index 883b1a4..84db347 100644 --- a/packages/runtime-node/src/server.ts +++ b/packages/runtime-node/src/server.ts @@ -57,7 +57,23 @@ import { type RedactOptions, } from "./redact.js"; import { startMemoryMetrics } from "./memory.js"; -import { configureRuntimeLogger, flushRuntimeLogs, shutdownRuntimeLogger, type RuntimeLoggingOptions } from "./logger.js"; +import { + configureRuntimeLogger, + currentOperation, + flushRuntimeLogs, + noteOperationError, + notifyException, + recordOperationAiUsage, + shutdownRuntimeLogger, + type RuntimeLoggingOptions, +} from "./logger.js"; +import { + expressRouteOf, + markCaptured, + setRequestExceptionCapturer, + startHookRequest, +} from "./requests.js"; +import { errorAttributes, errorInternal } from "@autter/runtime-core"; /** * Curated OpenTelemetry setup for Autter Runtime — deliberately NOT the @@ -78,10 +94,16 @@ import { configureRuntimeLogger, flushRuntimeLogs, shutdownRuntimeLogger, type R * request metrics : aggregated every 60 s, keyed by route * template (Express routes detected * out of the box — see captureExpressRoute) - * logs : not collected + * operation / request summaries: 100% (withRuntimeOperation, + * autterRequests, … — one wide + * record per unit of work, never + * sampled) + * logs : runtimeLogger / runtimeContext records + * (debug/info folded into the current + * operation's summary; minLevel applies) * * Exports OTLP/HTTP JSON to the Autter ingester (`/v1/traces`, - * `/v1/metrics`) with the ingest key as a bearer header. + * `/v1/metrics`, `/v1/logs`) with the ingest key as a bearer header. */ export interface AutterServerOptions { @@ -161,6 +183,8 @@ export interface LlmUsage { * built-in per-model price estimate. */ costUsd?: number; + /** Prompt tokens served from the provider cache (rolled into `autter.operation.ai`). */ + cacheReadTokens?: number; } export interface LlmCallInfo { @@ -578,11 +602,21 @@ async function runLlmSpan( kind: SpanKind.CLIENT, attributes: llmBaseAttributes(info), }); - const setResponseModel = (model: string) => + const usage: LlmUsage = {}; + let responseModel: string | undefined; + const setResponseModel = (model: string) => { + responseModel = model; span.setAttribute("gen_ai.response.model", model); + }; const handle: LlmCallHandle = { - setUsage: (usage) => span.setAttributes(usageAttributes(usage)), - setCost: (usd) => span.setAttribute("autter.llm.cost_usd", usd), + setUsage: (next) => { + Object.assign(usage, next); + span.setAttributes(usageAttributes(next)); + }, + setCost: (usd) => { + usage.costUsd = usd; + span.setAttribute("autter.llm.cost_usd", usd); + }, setResponseModel, setModel: setResponseModel, setAttributes: (attributes) => span.setAttributes(attributes), @@ -606,10 +640,12 @@ async function runLlmSpan( throw err; } finally { span.end(); + recordOperationAiUsage({ ...usage, model: responseModel ?? info.model }); } } function recordLlmCall(tracer: Tracer, call: TrackedLlmCall): void { + recordOperationAiUsage({ ...call, model: call.model }); const durationMs = Math.max(0, call.durationMs ?? 0); const endTime = new Date(); const startTime = new Date(endTime.getTime() - durationMs); @@ -642,8 +678,12 @@ async function runWithSpan( name: string, fn: () => T | Promise, attributes?: Attributes, + links?: Link[], ): Promise { - const span = tracer.startSpan(name, { attributes }); + const span = tracer.startSpan(name, { + attributes, + ...(links?.length ? { links } : {}), + }); try { const result = await context.with( trace.setSpan(context.active(), span), @@ -663,28 +703,6 @@ async function runWithSpan( } } -/** Express (4/5) assigns routing state onto the core request object; a - * matched handler leaves the route template on `req.route.path` and the - * mount prefix on `req.baseUrl`. */ -interface ExpressRequestProps { - baseUrl?: unknown; - route?: { path?: unknown }; -} - -/** - * Route template of a finished Express request ("/api/users/:id"), or null - * when no route matched (404s, static files) or the server isn't Express. - * Only meaningful at response end — Express fills `req.route` during routing. - */ -function expressRouteOf(req: unknown): string | null { - const props = req as ExpressRequestProps | null | undefined; - const path = props?.route?.path; - if (typeof path !== "string" || path === "") return null; - const base = typeof props?.baseUrl === "string" ? props.baseUrl : ""; - const route = base + path; - return route.startsWith("/") ? route : null; -} - /** * HttpInstrumentation responseHook: copy the Express route template into * `rpcMetadata.route` before the instrumentation snapshots attributes at @@ -713,6 +731,27 @@ function captureExpressRoute( // 'close' covers aborted ones; setRoute is idempotent. response.prependListener("finish", setRoute); response.prependListener("close", setRoute); + if (hookRequests) startHookRequest(response); +} + +/** `logging.requests: true` — zero-code request summaries (experimental). */ +let hookRequests = false; + +/** Span attributes for a structured error: autter.error.* (and the + * redacted, span-only `autter.error.internal`). */ +function structuredErrorAttributes(error: unknown): Attributes { + const attrs: Attributes = { ...errorAttributes(error, { causes: false }) }; + const internal = errorInternal(error); + if (internal) { + try { + attrs["autter.error.internal"] = JSON.stringify( + activeRedactor(internal as Attributes), + ).slice(0, 4000); + } catch { + /* unserialisable internal payload */ + } + } + return attrs; } let active: AutterServer | null = null; @@ -872,7 +911,26 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { const span = reuseActive ? (activeSpan as Span) : errorTracer.startSpan(isError ? error.name : "Error"); - span.setAttributes({ "autter.severity": "error", ...redacted }); + // Coded errors (RuntimeError, catalogs, or ANY error with + // code/why/fix/link/status/expected): autter.error.* on the span AND + // on the exception event (one span can carry several exceptions); + // cause chains on the event; `internal` on the span only. + const structured = structuredErrorAttributes(error); + const causes = errorAttributes(error); + const declared = Object.fromEntries( + Object.entries(causes).filter(([key]) => key !== "autter.error.internal"), + ); + const coded = Object.keys(declared).length > 0; + const requestId = currentOperation()?.requestId; + span.setAttributes({ + "autter.severity": "error", + ...redacted, + ...structured, + ...(requestId ? { "autter.request.id": requestId } : {}), + }); + markCaptured(error); + noteOperationError(error); + notifyException(error, redacted); if (isError && error.stack) { if (redacted["autter.handled"] === true) { @@ -882,6 +940,14 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { "exception.stacktrace": error.stack, "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true, + ...declared, + }); + } else if (coded) { + span.addEvent("exception", { + "exception.type": error.name, + "exception.message": message, + "exception.stacktrace": error.stack, + ...declared, }); } else { span.recordException(error); @@ -902,6 +968,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { ...(redacted["autter.handled"] === true ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } : {}), + ...declared, }); } span.setStatus({ code: SpanStatusCode.ERROR, message }); @@ -962,9 +1029,10 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { } configureRuntimeLogger({ endpoint, apiKey: options.apiKey, service: options.service, environment, release: options.release, options: options.logging, redact: activeRedactor, - run: (name, fn, attributes) => runWithSpan(processTracer, name, fn, activeRedactor(attributes)), + run: (name, fn, attributes, links) => runWithSpan(processTracer, name, fn, activeRedactor(attributes), links), reportOutcome, }); + hookRequests = options.logging?.requests === true; if (options.captureGlobalErrors !== false) { // `uncaughtExceptionMonitor` observes crashes WITHOUT changing the @@ -1020,6 +1088,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { trackLlmCall: (call) => recordLlmCall(llmTracer, call), shutdown: async () => { const logShutdown = await Promise.allSettled([shutdownRuntimeLogger()]); + hookRequests = false; stopMemoryMetrics?.(); active = null; activeAlwaysOnProvider = null; @@ -1046,6 +1115,13 @@ export function captureException( active.captureException(error, attributes); return; } + if (loggerOnlyCapture) { + loggerOnlyCapture(error, attributes); + return; + } + markCaptured(error); + noteOperationError(error); + notifyException(error, attributes); const span = trace .getTracer("autter-errors") .startSpan(error instanceof Error ? error.name : "Error", { attributes }); @@ -1057,6 +1133,23 @@ export function captureException( span.end(); } +/** @internal Set by initAutterLogging: exception capture without NodeSDK. */ +let loggerOnlyCapture: ((error: unknown, attributes?: Attributes) => void) | null = null; +/** @internal */ +export function setLoggerOnlyCapture( + fn: ((error: unknown, attributes?: Attributes) => void) | null, +): void { + loggerOnlyCapture = fn; +} +/** @internal True while initAutterServer's NodeSDK pipeline is active. */ +export function serverActive(): boolean { + return active !== null; +} +/** @internal Redacted structured-error span attributes (logger-only mode). */ +export { structuredErrorAttributes }; + +setRequestExceptionCapturer((error) => captureException(error)); + /** * Module-level convenience — see AutterServer.withProcessSpan. Before * initAutterServer runs it degrades to the global tracer, where the span is diff --git a/packages/runtime-node/src/sinks.ts b/packages/runtime-node/src/sinks.ts new file mode 100644 index 0000000..4f046f6 --- /dev/null +++ b/packages/runtime-node/src/sinks.ts @@ -0,0 +1,529 @@ +import { + appendFileSync, + mkdirSync, + readdirSync, + statSync, + unlinkSync, +} from "node:fs"; +import { join, resolve } from "node:path"; +import { + buildOtlpLogsRequest, + isSummary, + toOtlpLogRecord, + type OtlpLogRecord, + type RuntimeEvent, +} from "@autter/runtime-core"; + +/** + * Where runtime records go. The logger pipeline is + * `enrich → redact/bound → sinks[]`; each sink receives the same immutable + * RuntimeEvent. Defaults: `[otlpSink(), consoleSink()]` (+ `fileSink()` when + * NODE_ENV=development). Sinks must never throw into application code. + */ +export interface RuntimeSink { + readonly name: string; + /** Called when the logger is (re)configured; null before init. */ + start?(context: RuntimeSinkContext): void; + write(event: RuntimeEvent): void; + flush?(): Promise; + shutdown?(): Promise; + /** Records held in memory awaiting delivery. */ + buffered?(): number; + /** @internal Receives every captureException call (testing sinks). */ + exception?(record: CapturedException): void; +} + +export interface RuntimeSinkContext { + endpoint: string; + apiKey: string; + service: string; + environment: string; + release?: string; +} + +export interface CapturedException { + time: number; + error: unknown; + message: string; + type: string; + attributes: Record; + operationId?: string; + requestId?: string; +} + +/** Cumulative drop counter shared by every sink (runtimeLogStats().dropped). */ +export const sinkStats = { dropped: 0 }; + +// --------------------------------------------------------------------------- +// OTLP +// --------------------------------------------------------------------------- + +export interface OtlpSinkOptions { + /** Override the ingester base URL (default: the configured endpoint). */ + endpoint?: string; + /** Override the ingest key (default: the configured apiKey). */ + apiKey?: string; +} + +interface QueuedLog { + record: OtlpLogRecord; + bytes: number; +} + +/** + * OTLP/HTTP JSON export to `${endpoint}/v1/logs`. Bounded exactly like + * 1.4.0: 1000 records / 4 MiB buffered, 256 KiB per record, batches of 50 + * (≤ 512 KiB), 3 attempts within a 10 s deadline, exponential back-off + * on failure, undelivered records reported at shutdown. + */ +export function otlpSink(options: OtlpSinkOptions = {}): RuntimeSink { + let context: RuntimeSinkContext | null = null; + let queue: QueuedLog[] = []; + let queueBytes = 0; + let inFlightBytes = 0; + let inFlight = 0; + let timer: ReturnType | undefined; + let flushing: Promise | null = null; + let stopping = false; + let failureStreak = 0; + + const scheduleFlush = () => { + if (!timer && context && !stopping && queue.length) { + timer = setTimeout( + () => { + timer = undefined; + void flush().catch(() => {}); + }, + Math.min(30000, 2000 * 2 ** Math.min(failureStreak, 4)), + ); + timer.unref(); + } + }; + + const flush = async (): Promise => { + if (flushing) return flushing; + if (!context || !queue.length) return; + if (timer) { + clearTimeout(timer); + timer = undefined; + } + const active = context; + const endpoint = (options.endpoint ?? active.endpoint).replace(/\/$/, ""); + const apiKey = options.apiKey ?? active.apiKey; + const deadline = Date.now() + 10000; + flushing = (async () => { + const count = queue.length; + for (let remaining = count; remaining > 0; ) { + const batch: QueuedLog[] = []; + inFlightBytes = 0; + while (batch.length < Math.min(50, remaining) && queue.length) { + const next = queue[0]!; + if (batch.length && inFlightBytes + next.bytes > 512 * 1024) break; + batch.push(queue.shift()!); + inFlightBytes += next.bytes; + queueBytes -= next.bytes; + } + remaining -= batch.length; + inFlight = batch.length; + const body = JSON.stringify( + buildOtlpLogsRequest( + { + service: active.service, + environment: active.environment, + ...(active.release ? { release: active.release } : {}), + }, + batch.map((entry) => entry.record), + ), + ); + let failure: unknown; + for (let attempt = 0; attempt < 3; attempt++) { + try { + if (Date.now() >= deadline) + throw new Error("Runtime log flush exceeded 10 seconds"); + const response = await fetch(`${endpoint}/v1/logs`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${apiKey}`, + }, + body, + signal: AbortSignal.timeout( + Math.max(1, Math.min(3000, deadline - Date.now())), + ), + }); + if (!response.ok) + throw new Error(`Runtime log export failed (${response.status})`); + failure = undefined; + break; + } catch (error) { + failure = error; + } + } + inFlight = 0; + if (failure) { + failureStreak++; + queue = [...batch, ...queue]; + queueBytes += inFlightBytes; + inFlightBytes = 0; + throw failure; + } + inFlightBytes = 0; + failureStreak = 0; + } + })().finally(() => { + flushing = null; + scheduleFlush(); + }); + return flushing; + }; + + return { + name: "otlp", + start(next) { + context = next; + stopping = false; + failureStreak = 0; + }, + write(event) { + if (!context) return; + if (stopping) { + sinkStats.dropped++; + return; + } + const record = toOtlpLogRecord(event); + const bytes = Buffer.byteLength(JSON.stringify(record)); + if ( + bytes > 256 * 1024 || + queue.length + inFlight >= 1000 || + queueBytes + inFlightBytes + bytes > 4 * 1024 * 1024 + ) { + sinkStats.dropped++; + console.warn( + "[autter-runtime] log buffer or record limit reached; record dropped", + ); + return; + } + queue.push({ record, bytes }); + queueBytes += bytes; + scheduleFlush(); + }, + flush, + async shutdown() { + stopping = true; + try { + await flush(); + } finally { + if (queue.length) { + sinkStats.dropped += queue.length; + console.warn( + `[autter-runtime] ${queue.length} log records could not be delivered before shutdown`, + ); + } + queue = []; + queueBytes = 0; + context = null; + if (timer) clearTimeout(timer); + timer = undefined; + } + }, + buffered: () => queue.length + inFlight, + }; +} + +// --------------------------------------------------------------------------- +// Console +// --------------------------------------------------------------------------- + +export interface ConsoleSinkOptions { + /** Default: "pretty" unless NODE_ENV === "production", then "json". */ + format?: "pretty" | "json"; + /** Colourise pretty output. Default: stdout is a TTY and NO_COLOR is unset. */ + color?: boolean; + /** Line writer. Default console.log. */ + write?: (line: string) => void; +} + +const ANSI = { + dim: "\u001b[2m", + red: "\u001b[31m", + yellow: "\u001b[33m", + green: "\u001b[32m", + cyan: "\u001b[36m", + bold: "\u001b[1m", + reset: "\u001b[0m", +}; + +function shortValue(value: unknown): string { + if (typeof value === "string") return /\s/.test(value) ? JSON.stringify(value) : value; + if (value && typeof value === "object") { + const text = JSON.stringify(value); + return text.length > 120 ? `${text.slice(0, 117)}...` : text; + } + return String(value); +} + +/** `a.b=1 c=x` for user keys of a context tree (SDK-owned keys skipped). */ +function flatPairs(attributes: Record): string { + const pairs: string[] = []; + const visit = (prefix: string, value: unknown, depth: number) => { + if (pairs.length >= 24) return; + if (value && typeof value === "object" && !Array.isArray(value) && depth < 3) { + for (const [key, item] of Object.entries(value)) + visit(prefix ? `${prefix}.${key}` : key, item, depth + 1); + return; + } + pairs.push(`${prefix}=${shortValue(value)}`); + }; + for (const [key, value] of Object.entries(attributes)) { + if (/^(?:autter\.|exception\.|http\.(?:request\.method|route|response\.status_code)$)/.test(key)) + continue; + visit(key, value, 0); + } + return pairs.join(" "); +} + +/** Pretty one-line-plus-tree rendering used by consoleSink in development. */ +export function formatPretty(event: RuntimeEvent, color = false): string { + const paint = (code: keyof typeof ANSI, text: string) => + color ? `${ANSI[code]}${text}${ANSI.reset}` : text; + const a = event.attributes as Record; + const time = new Date(event.time).toISOString().slice(11, 23); + const levelColor = + event.level === "error" ? "red" : event.level === "warning" ? "yellow" : "dim"; + const level = paint(levelColor, event.level.toUpperCase().padEnd(7)); + if (!isSummary(event)) { + const context = flatPairs(a); + const op = typeof a["autter.operation.name"] === "string" ? paint("dim", ` [${a["autter.operation.name"]}]`) : ""; + return `${paint("dim", time)} ${level} ${event.message}${context ? ` ${paint("dim", context)}` : ""}${op}`; + } + const outcome = String(a["autter.operation.outcome"] ?? ""); + const outcomeColor = + outcome === "failed" ? "red" : outcome === "succeeded" ? "green" : "yellow"; + const status = a["http.response.status_code"]; + const head = [ + paint("dim", time), + level, + paint("bold", String(a["autter.operation.name"] ?? event.message)), + paint(outcomeColor, outcome), + ...(status !== undefined ? [String(status)] : []), + `${Math.round(Number(a["autter.operation.duration_ms"] ?? 0))}ms`, + ...(a["autter.request.id"] ? [paint("dim", `req=${a["autter.request.id"]}`)] : []), + ].join(" "); + const lines = [head]; + const row = (label: string, text: string) => + lines.push(` ${paint("cyan", label.padEnd(8))} ${text}`); + if (outcome !== "succeeded" && event.message && !event.message.endsWith(`: ${outcome}`)) + row("message", event.message); + const context = flatPairs(a); + if (context) row("context", context); + const steps = a["autter.operation.steps"]; + if (Array.isArray(steps) && steps.length) + row( + "steps", + steps + .map((step) => { + const s = step as { name?: string; status?: string; durationMs?: number }; + const mark = s.status === "failed" ? paint("red", "✗") : paint("green", "✓"); + return `${s.name} ${mark} ${s.durationMs}ms`; + }) + .join(" · "), + ); + const code = a["autter.error.code"]; + if (code || a["exception.type"]) + row( + "error", + [ + code ? paint("red", String(code)) : String(a["exception.type"]), + a["autter.error.expected"] === true ? paint("dim", "(expected)") : "", + a["autter.error.why"] ? `why: ${a["autter.error.why"]}` : "", + a["autter.error.fix"] ? `fix: ${a["autter.error.fix"]}` : "", + ] + .filter(Boolean) + .join(" "), + ); + const ai = a["autter.operation.ai"] as + | { calls?: number; input_tokens?: number; output_tokens?: number; cost_usd?: number; models?: string[] } + | undefined; + if (ai && typeof ai === "object") + row( + "ai", + `${ai.calls} call(s) · ${ai.input_tokens} in / ${ai.output_tokens} out · $${ai.cost_usd} · ${(ai.models ?? []).join(", ")}`, + ); + const logs = a["autter.operation.logs"]; + if (Array.isArray(logs)) + for (const entry of logs) { + const log = entry as { t?: number; level?: string; message?: string; attrs?: Record }; + const extra = log.attrs ? ` ${paint("dim", flatPairs(log.attrs))}` : ""; + row("log", `+${log.t}ms ${log.level} ${log.message}${extra}`); + } + if (a["autter.operation.logs_truncated"] === true) row("log", paint("dim", "… more messages truncated")); + return lines.join("\n"); +} + +/** + * Console output. JSON lines (identical to 1.4.0) in production; a compact + * human-readable tree elsewhere, coloured only on a TTY. + */ +export function consoleSink(options: ConsoleSinkOptions = {}): RuntimeSink { + const format = + options.format ?? + (process.env.NODE_ENV === "production" ? "json" : "pretty"); + const color = + options.color ?? + (process.stdout?.isTTY === true && !process.env.NO_COLOR); + const write = options.write ?? ((line: string) => console.log(line)); + return { + name: "console", + write(event) { + write( + format === "json" + ? JSON.stringify({ + ...event.attributes, + level: event.level, + message: event.message, + }) + : formatPretty(event, color), + ); + }, + }; +} + +// --------------------------------------------------------------------------- +// Local NDJSON files +// --------------------------------------------------------------------------- + +export interface FileSinkOptions { + /** Directory for `YYYY-MM-DD.jsonl` files. Default ".autter/runtime" (cwd-relative). */ + dir?: string; + /** Keep at most this many files (oldest deleted). Default 7. */ + maxFiles?: number; + /** Start a new `YYYY-MM-DD.N.jsonl` file past this size. Default 10 MiB. */ + maxBytes?: number; +} + +const FILE_NAME = /^(\d{4}-\d{2}-\d{2})(?:\.(\d+))?\.jsonl$/; + +/** + * NDJSON on local disk, one record per line, for coding agents and + * `autter logs --local`. Daily files (UTC), size rotation, bounded file + * count. Writes are synchronous appends (dev-sized volumes). Disables itself + * with one warning on read-only or permission-denied filesystems. + */ +export function fileSink(options: FileSinkOptions = {}): RuntimeSink { + const dir = resolve(options.dir ?? ".autter/runtime"); + const maxFiles = Math.max(1, options.maxFiles ?? 7); + const maxBytes = Math.max(1024, options.maxBytes ?? 10 * 1024 * 1024); + let context: RuntimeSinkContext | null = null; + let disabled = false; + let day = ""; + let index = 0; + let currentBytes = 0; + + const files = () => + readdirSync(dir) + .map((name) => ({ name, match: FILE_NAME.exec(name) })) + .filter((file) => file.match) + .map((file) => ({ + name: file.name, + day: file.match![1]!, + index: Number(file.match![2] ?? 0), + })) + .sort((a, b) => (a.day === b.day ? a.index - b.index : a.day < b.day ? -1 : 1)); + const fileName = (d: string, i: number) => (i === 0 ? `${d}.jsonl` : `${d}.${i}.jsonl`); + const prune = () => { + const current = fileName(day, index); + const existing = files().filter((file) => file.name !== current); + // The current file counts toward the limit even before its first write. + const excess = existing.length + 1 - maxFiles; + for (const file of existing.slice(0, Math.max(0, excess))) + unlinkSync(join(dir, file.name)); + }; + const open = (nextDay: string) => { + mkdirSync(dir, { recursive: true }); + day = nextDay; + const sameDay = files().filter((file) => file.day === day); + index = sameDay.length ? sameDay[sameDay.length - 1]!.index : 0; + try { + currentBytes = statSync(join(dir, fileName(day, index))).size; + } catch { + currentBytes = 0; + } + prune(); + }; + const disable = (error: unknown) => { + disabled = true; + const code = (error as { code?: string })?.code ?? "error"; + console.warn( + `[autter-runtime] local runtime files disabled (${code} writing ${dir}); set logging.file: false to silence`, + ); + }; + + return { + name: "file", + start(next) { + context = next; + }, + write(event) { + if (disabled) return; + try { + const line = `${JSON.stringify({ + time: new Date(event.time).toISOString(), + level: event.level, + message: event.message, + ...(context + ? { + service: context.service, + environment: context.environment, + ...(context.release ? { release: context.release } : {}), + } + : {}), + ...(event.traceId ? { traceId: event.traceId, spanId: event.spanId } : {}), + ...event.attributes, + })}\n`; + const bytes = Buffer.byteLength(line); + const today = new Date(event.time).toISOString().slice(0, 10); + if (today !== day) open(today); + if (currentBytes > 0 && currentBytes + bytes > maxBytes) { + index++; + currentBytes = 0; + prune(); + } + appendFileSync(join(dir, fileName(day, index)), line); + currentBytes += bytes; + } catch (error) { + disable(error); + } + }, + }; +} + +// --------------------------------------------------------------------------- +// Memory (testing) +// --------------------------------------------------------------------------- + +export interface MemorySink extends RuntimeSink { + readonly events: RuntimeEvent[]; + readonly exceptions: CapturedException[]; + clear(): void; +} + +/** Keeps every record in memory — for tests (`@autter/runtime-node/testing`). */ +export function memorySink(): MemorySink { + const events: RuntimeEvent[] = []; + const exceptions: CapturedException[] = []; + return { + name: "memory", + events, + exceptions, + write(event) { + events.push(event); + }, + exception(record) { + exceptions.push(record); + }, + clear() { + events.length = 0; + exceptions.length = 0; + }, + }; +} diff --git a/packages/runtime-node/src/testing.ts b/packages/runtime-node/src/testing.ts new file mode 100644 index 0000000..5a0cc6d --- /dev/null +++ b/packages/runtime-node/src/testing.ts @@ -0,0 +1,162 @@ +/** + * Test helpers — `@autter/runtime-node/testing`. No ingester, no network: + * every record the SDK would export is kept in memory. + * + * import { captureRuntime, expectOperation } from "@autter/runtime-node/testing"; + * + * const runtime = captureRuntime(); + * await request(app).post("/checkout").expect(402); + * expectOperation(runtime, "POST /checkout") + * .toHaveOutcome("degraded") + * .toHaveErrorCode("billing.declined"); + * runtime.stop(); + * + * Works with or without initAutterServer/initAutterLogging; before init it + * replaces the default console output instead of adding to it. + */ +import { isSummary, type RuntimeEvent, type RuntimeOutcome } from "@autter/runtime-core"; +import { addRuntimeSink } from "./logger.js"; +import { memorySink, type CapturedException } from "./sinks.js"; + +export { memorySink, type MemorySink, type CapturedException } from "./sinks.js"; +export type { RuntimeEvent }; + +export interface RuntimeCapture { + /** Every record, in emission order. */ + readonly events: RuntimeEvent[]; + /** Operation and request summaries. */ + readonly operations: RuntimeEvent[]; + /** Plain log records (warn/error and records outside operations). */ + readonly logs: RuntimeEvent[]; + /** Every captureException call (including boundary captures). */ + readonly exceptions: CapturedException[]; + /** All records stamped with this request id. */ + byRequestId(requestId: string): RuntimeEvent[]; + clear(): void; + /** Detach from the logger. */ + stop(): void; +} + +export function captureRuntime(): RuntimeCapture { + const sink = memorySink(); + const remove = addRuntimeSink(sink); + return { + get events() { + return [...sink.events]; + }, + get operations() { + return sink.events.filter(isSummary); + }, + get logs() { + return sink.events.filter((event) => !isSummary(event)); + }, + get exceptions() { + return [...sink.exceptions]; + }, + byRequestId: (requestId) => + sink.events.filter((event) => event.attributes["autter.request.id"] === requestId), + clear: () => sink.clear(), + stop: () => { + remove(); + }, + }; +} + +function partialMatch(actual: unknown, expected: unknown): boolean { + if (expected && typeof expected === "object" && !Array.isArray(expected)) { + if (!actual || typeof actual !== "object") return false; + return Object.entries(expected).every(([key, value]) => + partialMatch((actual as Record)[key], value), + ); + } + if (Array.isArray(expected)) + return ( + Array.isArray(actual) && + expected.length === actual.length && + expected.every((item, i) => partialMatch(actual[i], item)) + ); + return Object.is(actual, expected); +} + +class OperationExpectation { + constructor(readonly event: RuntimeEvent) {} + get attributes(): Record { + return this.event.attributes; + } + private fail(message: string): never { + const error = new Error( + `expectOperation(${JSON.stringify(this.attributes["autter.operation.name"])}): ${message}`, + ); + error.name = "AssertionError"; + throw error; + } + toHaveOutcome(outcome: RuntimeOutcome): this { + const actual = this.attributes["autter.operation.outcome"]; + if (actual !== outcome) this.fail(`expected outcome ${outcome}, got ${String(actual)}`); + return this; + } + toHaveKind(kind: "request" | "operation"): this { + const actual = this.attributes["autter.operation.kind"]; + if (actual !== kind) this.fail(`expected kind ${kind}, got ${String(actual)}`); + return this; + } + /** Deep partial match against the summary attributes. */ + toHaveContext(expected: Record): this { + if (!partialMatch(this.attributes, expected)) + this.fail(`context does not match ${JSON.stringify(expected)}`); + return this; + } + toHaveStep(name: string, status?: "succeeded" | "failed"): this { + const steps = (this.attributes["autter.operation.steps"] ?? []) as Array<{ name: string; status: string }>; + if (!steps.some((step) => step.name === name && (!status || step.status === status))) + this.fail(`no step ${name}${status ? ` (${status})` : ""}`); + return this; + } + toHaveErrorCode(code: string): this { + const actual = this.attributes["autter.error.code"]; + if (actual !== code) this.fail(`expected error code ${code}, got ${String(actual)}`); + return this; + } + toHaveLog(message: string | RegExp, level?: string): this { + const logs = (this.attributes["autter.operation.logs"] ?? []) as Array<{ message: string; level: string }>; + const ok = logs.some( + (log) => + (typeof message === "string" ? log.message === message : message.test(log.message)) && + (!level || log.level === level), + ); + if (!ok) this.fail(`no inline log ${String(message)}`); + return this; + } + toHaveRequestId(requestId?: string): this { + const actual = this.attributes["autter.request.id"]; + if (requestId === undefined ? typeof actual !== "string" : actual !== requestId) + this.fail(`expected request id ${requestId ?? "(any)"}, got ${String(actual)}`); + return this; + } +} + +/** + * Find the most recent summary named `name` (exact string or RegExp) and + * return chainable assertions. Throws when no such summary exists. + */ +export function expectOperation( + source: RuntimeCapture | RuntimeEvent[], + name: string | RegExp, +): OperationExpectation { + const events = Array.isArray(source) ? source.filter(isSummary) : source.operations; + const matches = events.filter((event) => { + const actual = String(event.attributes["autter.operation.name"] ?? ""); + return typeof name === "string" ? actual === name : name.test(actual); + }); + const event = matches[matches.length - 1]; + if (!event) { + const seen = events.map((e) => e.attributes["autter.operation.name"]); + const error = new Error( + `expectOperation: no operation named ${String(name)} (saw ${JSON.stringify(seen)})`, + ); + error.name = "AssertionError"; + throw error; + } + return new OperationExpectation(event); +} +export type { OperationExpectation }; diff --git a/packages/runtime-node/test/fixtures/golden-1.4.0-logs.json b/packages/runtime-node/test/fixtures/golden-1.4.0-logs.json new file mode 100644 index 0000000..cc90660 --- /dev/null +++ b/packages/runtime-node/test/fixtures/golden-1.4.0-logs.json @@ -0,0 +1,521 @@ +[ + { + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "golden" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "test" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "v1" + } + } + ] + }, + "scopeLogs": [ + { + "scope": { + "name": "autter-runtime" + }, + "logRecords": [ + { + "timeUnixNano": "