feat(sensor): Python runtime probe and secfoo instrument (#55)

benchmarks/sensor_overhead.py

@@ -0,0 +1,87 @@
+"""Overhead of the Python runtime probe (#55): requests/sec with and without it.
+
+    python benchmarks/sensor_overhead.py            # all modes, ~10 s
+    python benchmarks/sensor_overhead.py --requests 200000
+
+Each mode runs in a fresh interpreter. A "request" is a small handler that
+parses a query string, builds a response dict through a few app functions,
+and serializes it -- the shape of a typical web handler, without network
+noise. "watched" calls the watched vulnerable function on every request
+(the worst case); "unwatched" never calls it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import textwrap
+from pathlib import Path
+
+APP = textwrap.dedent("""\
+    import json, sys, time, urllib.parse
+    import vulnlib
+
+    def parse(qs):
+        return dict(urllib.parse.parse_qsl(qs))
+
+    def build(params, call_vuln):
+        body = {"user": params.get("user"), "items": [i * 2 for i in range(10)]}
+        if call_vuln:
+            body["parsed"] = vulnlib.load(params.get("doc", ""))
+        return body
+
+    def handle(qs, call_vuln):
+        return json.dumps(build(parse(qs), call_vuln))
+
+    n, call_vuln = int(sys.argv[1]), sys.argv[2] == "watched"
+    for _ in range(2000):
+        handle("user=a&doc=x", call_vuln)
+    start = time.perf_counter()
+    for _ in range(n):
+        handle("user=alice&doc=hello", call_vuln)
+    print(n / (time.perf_counter() - start))
+    """)
+VULNLIB = "def load(text):\n    return text[::-1]\n"
+
+
+def run(workdir: Path, requests: int, traffic: str, env_extra: dict[str, str] | None) -> float:
+    env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")}
+    for key in ("SECFOO_SENSOR", "SECFOO_SENSOR_ADDR", "SECFOO_EVENTS_OUT"):
+        env.pop(key, None)
+    env.update(env_extra or {})
+    cmd = [sys.executable, "-m", "secfoo.sensor", "app.py", str(requests), traffic]
+    out = subprocess.run(cmd, cwd=workdir, env=env, capture_output=True, text=True, check=True)  # nosec B603
+    return float(out.stdout.strip().splitlines()[-1])
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--requests", type=int, default=100_000)
+    args = parser.parse_args()
+    with tempfile.TemporaryDirectory() as tmp:
+        workdir = Path(tmp)
+        (workdir / "app.py").write_text(APP, encoding="utf-8")
+        (workdir / "vulnlib.py").write_text(VULNLIB, encoding="utf-8")
+        (workdir / "watchlist.json").write_text(json.dumps({"version": 1, "generated_at": "x", "entries": [{
+            "finding_fingerprint": "fp", "advisory": None, "ecosystem": "PyPI", "package": "vulnlib",
+            "version": "1", "granularity": "symbol", "symbols": ["vulnlib.load"], "source": "osv"}]}), encoding="utf-8")
+        probe_env = {"SECFOO_SENSOR": "1", "SECFOO_WATCHLIST": "watchlist.json",
+                     "SECFOO_EVENTS_OUT": str(workdir / "events.ndjson"), "SECFOO_FLUSH_INTERVAL": "5"}
+        backends = ["profile"] + (["monitoring"] if hasattr(sys, "monitoring") else [])
+        print(f"Python {sys.version.split()[0]}, {args.requests:,} requests per mode\n")
+        print(f"{'mode':<34}{'req/s':>12}{'overhead':>10}")
+        for traffic in ("unwatched", "watched"):
+            base = run(workdir, args.requests, traffic, None)
+            print(f"{'no probe, ' + traffic:<34}{base:>12,.0f}{'':>10}")
+            for backend in backends:
+                rate = run(workdir, args.requests, traffic, {**probe_env, "SECFOO_SENSOR_BACKEND": backend})
+                label = f"probe ({backend}), {traffic}"
+                print(f"{label:<34}{rate:>12,.0f}{(base / rate - 1) * 100:>9.1f}%")
+
+
+if __name__ == "__main__":
+    main()

docs/REFERENCE.md

@@ -473,6 +473,73 @@ runtime-confirmed findings, but only once there is runtime data.
 
 ![SCA vulnerabilities list with the Runtime column and a highlighted conflict](assets/sca-runtime.jpg)
 
+### Python probe
+
+The probe runs inside a Python app and records which watched vulnerable
+functions actually run. It writes the event format above.
+
+```bash
+secfoo runtime watchlist --project payments -o watchlist.json
+secfoo instrument --watchlist watchlist.json -o events.ndjson -- python app.py   # or gunicorn ..., uvicorn ...
+secfoo runtime ingest events.ndjson --project payments
+```
+
+`secfoo instrument` starts the probe in every Python process the command
+launches, workers included, without changing the app's start command. It
+does this through a temporary `sitecustomize` on `PYTHONPATH`. When you can't
+wrap the command, set the variables yourself:
+
+```bash
+SECFOO_SENSOR=1 SECFOO_WATCHLIST=watchlist.json SECFOO_EVENTS_OUT=events-{pid}.ndjson python -m secfoo.sensor app.py
+```
+
+| Variable | Meaning |
+|---|---|
+| `SECFOO_SENSOR=1` | Required. Without it the probe installs nothing: no hooks, threads or connections. |
+| `SECFOO_WATCHLIST` | The watchlist file. It is re-read when its mtime changes (checked at most every 30 s). |
+| `SECFOO_EVENTS_OUT` | The events file. `{pid}` gives each process its own file; otherwise processes append to one file safely. |
+| `SECFOO_SENSOR_ADDR` | Send to a sensor instead: `unix://…`, `tcp://127.0.0.1:port` (loopback only), `npipe://…`, `https://…` (direct mode, using the runtime ingest API with `SECFOO_API_KEY_FILE` / `SECFOO_PROJECT`), or `file://…`. |
+| `SECFOO_FLUSH_INTERVAL` | Seconds between flushes (default 10). It also flushes at exit. |
+| `SECFOO_APP_ROOT` | The app's own code, for coverage and stack frames (default: the working directory). |
+
+**How it works.**
+
+- **Backends.** On Python 3.12+ it uses `sys.monitoring` (PEP 669), with
+  `PY_START` events only on the watched functions' code objects, so unwatched
+  code costs nothing. On 3.10/3.11 it falls back to `sys.setprofile` and
+  `threading.setprofile`.
+- **Symbols** are resolved as their modules are imported.
+- **Coverage.** Each app function is counted the first time it runs, for
+  `app_functions_executed`. On 3.12+ that event is then disabled for the
+  function.
+- **Inventory.** It emits `load` lines for installed distributions whose
+  top-level module was imported.
+- **Workers.** Each process, including forked or spawned workers (gunicorn,
+  uvicorn, multiprocessing), is its own session. With gunicorn `--preload`,
+  the probe starts in the master and each forked worker starts a new session
+  automatically.
+
+**Guarantees.**
+
+- It never reads `frame.f_locals`, arguments or return values. It records
+  only function identity, counts, timestamps and `file:line` app frames.
+- An internal error disables the probe, with one stderr line, and never
+  reaches the app.
+- Transports run on a background thread and drop data rather than block
+  when the destination is gone.
+- It opens no connection other than `SECFOO_SENSOR_ADDR`.
+
+**Overhead** (`python benchmarks/sensor_overhead.py`, CPython 3.14, a ~7 µs
+request-like handler):
+
+| Mode | Unwatched code | Watched function called on every request |
+|---|---|---|
+| `sys.monitoring` (3.12+) | ≈ 0% | ~20% (≈ 1.4 µs per watched call) |
+| `setprofile` fallback (3.10/3.11) | ~10× slower | ~10× slower |
+
+The fallback fires on every Python call, so prefer 3.12+ in production or
+keep the fallback to staging.
+
 ### Runtime ingest API
 
 Live sensors send evidence to the same server your CLI results go to. They

src/secfoo/cli.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import os
 import shutil
 import sys
 from datetime import date, datetime, timezone
@@ -1815,6 +1816,67 @@
     sys.stdout.write(render_prompt(skill, context, depth=depth.value) + "\n")
 
 
+_SITECUSTOMIZE = """\
+# Written by `secfoo instrument` (#55): starts the secfoo runtime probe in
+# every Python process of the wrapped command. Inert unless SECFOO_SENSOR=1.
+try:
+    import secfoo.sensor as _secfoo_sensor
+    _secfoo_sensor.start_from_env()
+except Exception:  # never break the app's interpreter start-up
+    pass
+"""
+
+
+@app.command(
+    name="instrument",
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+)
+def instrument(
+    ctx: typer.Context,
+    watchlist: Annotated[Path | None, typer.Option(
+        "--watchlist", exists=True, dir_okay=False, help="Watchlist from `secfoo runtime watchlist`.",
+    )] = None,
+    output: Annotated[Path | None, typer.Option(
+        "--output", "-o",
+        help="Write events here ({pid} is replaced per process). Ignored when SECFOO_SENSOR_ADDR is set.",
+    )] = None,
+    flush_interval: Annotated[float, typer.Option("--flush-interval", min=0.05)] = 10.0,
+) -> None:
+    """Run a Python command with the secfoo runtime probe: secfoo instrument -o events.ndjson -- python app.py
+
+    The probe starts in every Python process the command launches (workers
+    included), records which watched vulnerable functions run, and writes
+    #53 events. Then: `secfoo runtime ingest events.ndjson --project <p>`.
+    """
+    import subprocess  # nosec B404 -- runs the user's own command, as given
+    import tempfile
+
+    import secfoo
+
+    command = list(ctx.args)
+    if command[:1] == ["--"]:
+        command = command[1:]
+    if not command:
+        err_console.print("[red]Error:[/] give the command to run after --, e.g. -- python app.py")
+        raise typer.Exit(code=2)
+    env = dict(os.environ)
+    if output is None and not env.get("SECFOO_SENSOR_ADDR") and not env.get("SECFOO_EVENTS_OUT"):
+        err_console.print("[red]Error:[/] pass -o events.ndjson (or set SECFOO_SENSOR_ADDR) so events go somewhere.")
+        raise typer.Exit(code=2)
+    with tempfile.TemporaryDirectory(prefix="secfoo-instrument-") as hook_dir:
+        Path(hook_dir, "sitecustomize.py").write_text(_SITECUSTOMIZE, encoding="utf-8")
+        package_root = str(Path(secfoo.__file__).resolve().parent.parent)
+        env["PYTHONPATH"] = os.pathsep.join(p for p in (hook_dir, package_root, env.get("PYTHONPATH", "")) if p)
+        env["SECFOO_SENSOR"] = "1"
+        env["SECFOO_FLUSH_INTERVAL"] = str(flush_interval)
+        if watchlist is not None:
+            env["SECFOO_WATCHLIST"] = str(watchlist.resolve())
+        if output is not None:
+            env["SECFOO_EVENTS_OUT"] = str(output.resolve())
+        result = subprocess.run(command, env=env, check=False)  # nosec B603 -- the user's command
+    raise typer.Exit(code=result.returncode)
+
+
 MERMAID_CDN_URL = "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js"
 
 

src/secfoo/sensor/__init__.py

@@ -0,0 +1,49 @@
+"""secfoo runtime sensor: the in-process Python probe (#55).
+
+Opt-in only (#51 rule 0). `start_from_env()` does nothing unless
+`SECFOO_SENSOR=1` is set, so importing this package -- or having it on
+PYTHONPATH -- never changes an app's behavior.
+
+    SECFOO_SENSOR=1 SECFOO_WATCHLIST=watchlist.json SECFOO_EVENTS_OUT=events.ndjson python -m secfoo.sensor app.py
+    secfoo instrument --watchlist watchlist.json -o events.ndjson -- python app.py
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+_probe = None
+
+
+def enabled(environ=None) -> bool:
+    return (os.environ if environ is None else environ).get("SECFOO_SENSOR", "").strip() == "1"
+
+
+def start_from_env():
+    """Start the probe from SECFOO_* variables. Returns it, or None when the
+    sensor isn't enabled. Never raises into the app."""
+    global _probe
+    if _probe is not None or not enabled():
+        return _probe
+    try:
+        from secfoo.sensor.probe import DEFAULT_FLUSH_SECONDS, Probe
+        from secfoo.sensor.transport import from_env
+
+        flush = float(os.environ.get("SECFOO_FLUSH_INTERVAL", DEFAULT_FLUSH_SECONDS))
+        _probe = Probe(
+            watchlist_path=os.environ.get("SECFOO_WATCHLIST") or None,
+            transport=from_env(),
+            app_root=os.environ.get("SECFOO_APP_ROOT") or None,
+            flush_seconds=max(flush, 0.05),
+            backend=os.environ.get("SECFOO_SENSOR_BACKEND") or None,
+        )
+        _probe.start()
+    except Exception as exc:  # noqa: BLE001 -- fail-open: a bad config never stops the app
+        print(f"secfoo sensor: not started ({exc})", file=sys.stderr)
+        _probe = None
+    return _probe
+
+
+def current_probe():
+    return _probe

src/secfoo/sensor/__main__.py

@@ -0,0 +1,29 @@
+"""`python -m secfoo.sensor app.py [args]` or `python -m secfoo.sensor -m module [args]`.
+
+Starts the probe (only if SECFOO_SENSOR=1), then runs the app exactly as
+`python app.py` would.
+"""
+
+from __future__ import annotations
+
+import runpy
+import sys
+
+from secfoo.sensor import start_from_env
+
+
+def main(argv: list[str]) -> None:
+    if not argv:
+        print("usage: python -m secfoo.sensor app.py [args] | -m module [args]", file=sys.stderr)
+        raise SystemExit(2)
+    start_from_env()
+    if argv[0] == "-m" and len(argv) > 1:
+        sys.argv = [argv[1], *argv[2:]]
+        runpy.run_module(argv[1], run_name="__main__", alter_sys=True)
+    else:
+        sys.argv = list(argv)
+        runpy.run_path(argv[0], run_name="__main__")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])