Redact authentication material from application telemetry

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 Application releases and Alembic schema revisions are separate identifiers.
 
+## Unreleased
+
+### Fixes
+
+- Keep sign-in codes, state values, and authentication credentials out of request logs and browser diagnostics.
+
 ## 0.0.50 — 2026-10-03
 
 ### Enhancements

apps/web/tests/telemetry-browser-faro.test.ts

@@ -8,7 +8,7 @@ import {
 } from "@grafana/faro-web-sdk";
 import { normalizeMeta, normalizePayload } from "@devfeed/telemetry/privacy";
 
-it("telemetry preserves Faro sampling and original event and error content", () => {
+it("telemetry preserves Faro sampling and diagnostics while removing authentication values", () => {
   const delivered: TransportItem[] = [];
   class Capture extends BaseTransport {
     name = "test-capture";
@@ -44,6 +44,10 @@ it("telemetry preserves Faro sampling and original event and error content", ()
   faro.api.setUser({ email: "private-email", id: "private-user" });
   faro.api.pushEvent("telemetry_ready", { url: "/search?q=original-query" });
   faro.api.pushError(new Error("original-error-message"));
+  faro.api.pushEvent("route_change", {
+    url: "/api/v1/user/auth/callback?code=private-code&state=private-state&limit=25",
+  });
+  faro.api.pushError(new Error("GET /auth/callback?access_token=private-token failed"));
   expect(
     delivered.some(
       (item) =>
@@ -53,7 +57,8 @@ it("telemetry preserves Faro sampling and original event and error content", ()
   expect(JSON.stringify(delivered)).toContain("private-email");
   expect(JSON.stringify(delivered)).toContain("original-error-message");
   expect(JSON.stringify(delivered)).toContain("/search?q=original-query");
-  expect(JSON.stringify(delivered)).not.toContain("redacted");
+  expect(JSON.stringify(delivered)).not.toMatch(/private-code|private-state|private-token/);
+  expect(JSON.stringify(delivered)).toContain("code=[REDACTED]&state=[REDACTED]&limit=25");
   expect(JSON.stringify(delivered)).not.toContain("isSampled");
   faro.instrumentations.remove(...faro.instrumentations.instrumentations);
 });

apps/web/tests/telemetry.test.ts

@@ -3,12 +3,96 @@ import { traceHeaders } from "@devfeed/telemetry/propagation";
 import { describe, expect, it, vi } from "vitest";
 import { createServer } from "node:http";
 import { once } from "node:events";
-import { routeName, normalizeBody, normalizePayload } from "@devfeed/telemetry/privacy";
+import {
+  routeName,
+  normalizeBody,
+  normalizePayload,
+  redactAuthText,
+} from "@devfeed/telemetry/privacy";
+import redactionCases from "../../../packages/telemetry/fixtures/auth-redaction-cases.json";
 import { receiveTelemetry } from "@devfeed/telemetry/receiver";
 import { registerTelemetry } from "@devfeed/telemetry/server";
 const settings = { enabled: true, app: "web" as const, version: "test", environment: "test" };
 
 describe("operational telemetry", () => {
+  it.each(redactionCases)("redacts authentication in $name", ({ input, expected }) => {
+    expect(redactAuthText(input)).toBe(expected);
+    expect(redactAuthText(expected)).toBe(expected);
+  });
+  it("redacts nested Faro URLs, credentials and OTLP attributes while preserving correlation", () => {
+    const callback =
+      "https://devfeed.tech/auth/callback?code=private-code&state=private-state&q=react";
+    const input = {
+      meta: { page: { url: callback }, session: { id: "session-123" } },
+      exceptions: [
+        { value: `GET ${callback} failed`, stacktrace: { frames: [{ filename: callback }] } },
+      ],
+      events: [{ name: "route_change", attributes: { url: callback, token: "private-token" } }],
+      logs: [{ message: callback }],
+      measurements: [{ type: "web-vitals", values: { lcp: 123 } }],
+      traces: {
+        resourceSpans: [
+          {
+            scopeSpans: [
+              {
+                spans: [
+                  {
+                    traceId: "a".repeat(32),
+                    spanId: "b".repeat(16),
+                    status: { code: 2 },
+                    attributes: [
+                      { key: "http.url", value: { stringValue: callback } },
+                      {
+                        key: "url.query",
+                        value: { stringValue: "code=private-code&state=private-state" },
+                      },
+                      {
+                        key: "http.request.header.authorization",
+                        value: { stringValue: "Bearer private-token" },
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      },
+    };
+    const result = normalizeBody(input, settings);
+    expect(JSON.stringify(result)).not.toMatch(/private-code|private-state|private-token/);
+    expect(result.meta).toMatchObject({
+      page: { url: callback.replaceAll(/private-(?:code|state)/g, "[REDACTED]") },
+      session: { id: "session-123" },
+    });
+    expect(result.measurements[0]).toMatchObject({ values: { lcp: 123 } });
+    expect(result.traces).toMatchObject({
+      resourceSpans: [
+        {
+          scopeSpans: [
+            {
+              spans: [
+                {
+                  traceId: "a".repeat(32),
+                  spanId: "b".repeat(16),
+                  status: { code: 2 },
+                  attributes: [
+                    { key: "http.url" },
+                    { key: "url.query" },
+                    {
+                      key: "http.request.header.authorization",
+                      value: { stringValue: "[REDACTED]" },
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    });
+    expect(JSON.stringify(input)).toContain("private-code");
+  });
   it("bounds unknown routes, slugs, query strings and dynamic API paths", () => {
     const routes = new Set(
       Array.from({ length: 1000 }, (_, i) => routeName(`/articles/private-${i}?token=secret`)),
@@ -32,7 +116,7 @@ describe("operational telemetry", () => {
       attributes: { url: "/content/articles/private-id" },
     });
   });
-  it("preserves exception messages, frames, console logs and metadata", () => {
+  it("preserves diagnostics and metadata except authentication fields", () => {
     const result = normalizeBody(
       {
         meta: {
@@ -77,7 +161,7 @@ describe("operational telemetry", () => {
     expect(result.exceptions[0]).toMatchObject({
       type: "secret",
       value: "secret",
-      context: { secret: "secret" },
+      context: { secret: "[REDACTED]" },
       stacktrace: {
         frames: [
           {
@@ -89,7 +173,7 @@ describe("operational telemetry", () => {
         ],
       },
     });
-    expect(JSON.stringify(result)).not.toMatch(/redacted|omitted/);
+    expect(JSON.stringify(result)).not.toContain("omitted");
   });
   it("preserves trace correlation, timing and original OTLP fields", () => {
     const result = normalizePayload("trace", {
@@ -154,8 +238,14 @@ describe("operational telemetry", () => {
             await receiveTelemetry(
               request(
                 JSON.stringify({
-                  meta: { user: { email: "secret" } },
-                  logs: [{ message: "secret" }],
+                  meta: {
+                    user: { email: "secret" },
+                    page: { url: "/auth/callback?code=private-code&state=private-state" },
+                  },
+                  logs: [
+                    { message: "secret" },
+                    { message: "/auth/callback?access_token=private-token" },
+                  ],
                 }),
               ),
               app,
@@ -164,7 +254,11 @@ describe("operational telemetry", () => {
         ).toBe(202);
         const options = upstream.mock.calls[0][1];
         expect(options.headers["X-API-Key"]).toBe("server-key");
-        expect(JSON.parse(options.body).logs).toEqual([{ message: "secret" }]);
+        expect(JSON.parse(options.body).logs).toEqual([
+          { message: "secret" },
+          { message: "/auth/callback?access_token=[REDACTED]" },
+        ]);
+        expect(options.body).not.toMatch(/private-code|private-state|private-token/);
         expect(JSON.stringify(options.headers)).not.toContain("secret-cookie");
         upstream.mockRejectedValueOnce(new Error("private-collector-error"));
         expect((await receiveTelemetry(request("{}"), app)).status).toBe(503);
@@ -201,7 +295,18 @@ describe("operational telemetry", () => {
       expect((await fetch(`http://127.0.0.1:${appPort}/`)).status).toBe(200);
       await fetch(`http://127.0.0.1:${appPort}/articles/item-123?token=original-value`);
       expect(log.mock.calls.map(([line]) => JSON.parse(line))).toContainEqual(
-        expect.objectContaining({ route: "/articles/item-123?token=original-value" }),
+        expect.objectContaining({ route: "/articles/item-123?token=[REDACTED]" }),
+      );
+      await fetch(
+        `http://127.0.0.1:${appPort}/api/v1/user/auth/callback?code=private-code&state=private-state&limit=25`,
+      );
+      expect(log.mock.calls.map(([line]) => JSON.parse(line))).toContainEqual(
+        expect.objectContaining({
+          route: "/api/v1/user/auth/callback?code=[REDACTED]&state=[REDACTED]&limit=25",
+        }),
+      );
+      expect(JSON.stringify(log.mock.calls)).not.toMatch(
+        /original-value|private-code|private-state/,
       );
       expect((await fetch(`http://127.0.0.1:${port}/`)).status).toBe(404);
       const response = await fetch(`http://127.0.0.1:${port}/metrics`);

packages/core/OBSERVABILITY.md

@@ -125,21 +125,29 @@ this change covers the Python services.
 
 ## Application log content
 
-Text, JSON and stored job logs no longer redact supplied values. Request URLs keep
-query values; structured fields, nested values, library messages and exception
-messages are preserved. Text summaries remain concise, with full context at DEBUG.
+Text and JSON logs remove authentication material at their output boundary.
+Request URLs retain concrete paths, IDs and ordinary query values, while codes,
+state, tokens, client secrets and other authentication parameters become
+`[REDACTED]`. Matching is case-insensitive and covers encoded parameter names,
+repeated parameters, fragments and encoded nested URLs. URL credentials and
+credential fields in structured context, library messages and exception messages
+are also removed, including Bearer and Basic authorization values. Ordinary
+search/filter values, status codes, request IDs and trace correlation remain
+available. Text summaries remain concise, with full context at DEBUG.
 Request URLs and log messages have no string-length cap. Control characters are
 escaped; job-stream event size, queue and retention limits remain.
 The logger does not automatically capture request bodies, headers or local variables.
 
-Browser telemetry retains original error messages, stack filenames/functions,
-console log payloads, event attributes and metadata. Browser and Node trace exports
-retain supplied span attributes and events. The browser receiver still validates
+Browser telemetry removes the same authentication material from error messages,
+stack filenames, log payloads, event attributes, metadata and OTLP attributes.
+Sanitization runs before browser transport and again at the receiver, including
+when a client skips browser normalization. Node request logs use the same URL policy.
+Other diagnostic content is preserved. The browser receiver still validates
 origin, content type and size, enforces rate/time limits, and assigns service identity.
 Metric routes remain grouped to limit cardinality; Node request logs use concrete URLs.
 
-Restart running services to load the updated logging code. Previously masked values
-in retained records cannot be reconstructed. This change needs no database migration.
+Restart running services to load the updated logging code. Redaction applies to new
+records; it does not remove authentication material from existing retained logs.
 
 ## Migration
 

packages/core/src/http_logging.py

@@ -1,10 +1,12 @@
-"""Concrete request URLs for logs, with all supplied query values retained."""
+"""Concrete request URLs for logs, excluding authentication material."""
 
 import re
 from collections.abc import Mapping
 from typing import Any
 from urllib.parse import quote
 
+from devfeed_core.log_privacy import redact_auth_text
+
 _HOST = re.compile(r"(?:[a-z0-9.-]+|\[[a-f0-9:.]+\])(?::[0-9]+)?\Z", re.IGNORECASE)
 
 
@@ -38,5 +40,5 @@ def request_log_fields(scope: Mapping[str, Any]) -> dict[str, str]:
     return {
         "method": scope["method"],
         "route": path,
-        "request_url": url + ("?" + query if query else ""),
+        "request_url": redact_auth_text(url + ("?" + query if query else "")),
     }

packages/core/src/log_privacy.py

@@ -0,0 +1,99 @@
+"""Remove authentication material without hiding ordinary operational context."""
+
+import re
+from urllib.parse import quote, unquote
+
+REDACTED = "[REDACTED]"
+# These names are credentials even outside a URL. Avoid ambiguous fields such as
+# status.code and application state; code/state are sensitive query parameters.
+_CREDENTIAL_FIELDS = frozenset(
+    {
+        "accesstoken",
+        "refreshtoken",
+        "idtoken",
+        "token",
+        "clientsecret",
+        "codeverifier",
+        "assertion",
+        "clientassertion",
+        "password",
+        "passwd",
+        "secret",
+        "apikey",
+        "authorization",
+        "cookie",
+        "setcookie",
+        "samlresponse",
+        "oauthtoken",
+        "oauthverifier",
+        "devicecode",
+        "usercode",
+    }
+)
+_QUERY_FIELDS = _CREDENTIAL_FIELDS | {"code", "state", "sessionstate", "relaystate"}
+_PARAMETER = re.compile(r"(^|[?&#;\s])([\w%+.-]+)=([^&#;\s\"'<>]*)")
+_USERINFO = re.compile(r"(\b[a-z][a-z0-9+.-]*://)[^/?#\s@]+@", re.IGNORECASE)
+_AUTHORIZATION = re.compile(r"(\b(?:bearer|basic)\s+)[a-z0-9._~+/-]+=*", re.IGNORECASE)
+
+
+def _name(value: str) -> str:
+    return re.sub(r"[_.-]", "", unquote(value).casefold())
+
+
+def _credential_field(value: str) -> bool:
+    return _name(value.rsplit(".", 1)[-1]) in _CREDENTIAL_FIELDS
+
+
+def _encoded_value(value: str, depth: int) -> str:
+    decoded = unquote(value)
+    if decoded == value:
+        return value
+    if depth >= 5:
+        return REDACTED
+    sanitized = _encoded_value(decoded, depth + 1)
+    if "?" in sanitized or "#" in sanitized or "://" in sanitized:
+        sanitized = redact_auth_text(sanitized, depth + 1)
+    return quote(sanitized, safe="") if sanitized != decoded else value
+
+
+def redact_auth_text(value: str, depth: int = 0) -> str:
+    """Keep URL spelling/ordering intact except credentials, including nested URLs."""
+    value = _AUTHORIZATION.sub(lambda match: match[1] + REDACTED, value)
+    if "=" not in value and "@" not in value:
+        return value
+    value = _USERINFO.sub(lambda match: match[1] + REDACTED + "@", value)
+
+    def parameter(match: re.Match[str]) -> str:
+        prefix, key, original = match.groups()
+        if _name(key) in _QUERY_FIELDS:
+            return prefix + key + "=" + REDACTED
+        if "%" in original:
+            return prefix + key + "=" + _encoded_value(original, depth)
+        return match[0]
+
+    return _PARAMETER.sub(parameter, value)
+
+
+def redact_authentication(value, depth: int = 0):
+    """Protect structured fields and URL-bearing messages at the output boundary."""
+    if isinstance(value, str):
+        return redact_auth_text(value)
+    if isinstance(value, (dict, list)) and depth >= 20:
+        return "[truncated]"
+    if isinstance(value, dict):
+        output = {
+            key: REDACTED if _credential_field(str(key)) else redact_authentication(item, depth + 1)
+            for key, item in value.items()
+        }
+        if (
+            isinstance(value.get("key"), str)
+            and "value" in value
+            and _credential_field(value["key"])
+        ):
+            output["value"] = (
+                {"stringValue": REDACTED} if isinstance(value["value"], dict) else REDACTED
+            )
+        return output
+    if isinstance(value, list):
+        return [redact_authentication(item, depth + 1) for item in value]
+    return value