From 0caf99862b2a0b5df737341d3f131c682d88b8b9 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:35:45 -0400 Subject: [PATCH 1/2] fix: release Panel views left behind by destroyed sessions Every closed session leaked in production: Panel registers the per-session NotificationArea in pn.state._views and only removes it via state._notifications[doc], which Panel's own Document.destroy override deletes before Bokeh runs session-destroyed callbacks. The orphaned entry pins the dead Document and often the whole ChannelmapGUI (~90k Electrode objects), so RSS climbed to ~3.4 GB until the container was OOM-killed. Register _release_dead_session_views on session destroy to drop every _views entry still pointing at the dead document. Under the same load idle RSS after cleanup falls from ~1.3-1.4 GB to ~660 MB. Tests: an end-to-end Panel server test (fails before, passes after), unit tests for the hook's edge cases, and a strict-xfail canary that starts failing once Panel fixes the bug so the workaround can go. Refs #6 Assisted-by: ClaudeCode:claude-opus-5-5 --- pixelmap/gui/gui.py | 32 ++++++ tests/test_session_leak.py | 203 +++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 tests/test_session_leak.py diff --git a/pixelmap/gui/gui.py b/pixelmap/gui/gui.py index 14c3a21..5f496f5 100644 --- a/pixelmap/gui/gui.py +++ b/pixelmap/gui/gui.py @@ -3166,6 +3166,37 @@ def _extract_cfg_from_session_args() -> str | None: return value or None +def _release_dead_session_views(session_context): + """Drop every ``pn.state._views`` entry still pointing at a dead session. + + Works around a Panel bug (1.7.5 through at least 1.9.4) that leaked every + closed session in production. Panel registers the per-session + ``NotificationArea`` in ``state._views``, a plain dict holding + ``(view, root, doc, comm)``, and only removes it from the notification + area's ``_server_destroy`` -- which Panel reaches through + ``state._notifications[doc]``. But Panel's ``Document.destroy`` override + deletes ``state._notifications[doc]`` first, and Bokeh destroys the + document *before* running session-destroyed callbacks, so that hook never + fires. The orphaned entry pins the dead Document and, through its models' + callbacks, this session's whole ``ChannelmapGUI`` (~90k ``Electrode`` + objects of wiring maps). RSS climbed to ~3.4 GB and the container was + OOM-killed. + + Order-independent with Panel's own hooks: they skip refs already gone. + """ + doc = getattr(session_context, "_document", None) + if doc is None: + return + for ref, (view, root, view_doc, _comm) in list(pn.state._views.items()): + if view_doc is not doc: + continue + try: + view._cleanup(root) + except Exception as exc: # noqa: BLE001 - a failed cleanup must not keep the session pinned + print(f"View cleanup failed for {type(view).__name__}: {exc}") + pn.state._views.pop(ref, None) + + def create_app(): """Create and configure the Panel app""" # Re-arm the extensions for *this* session. Panel keys `config` on the @@ -3194,6 +3225,7 @@ def create_app(): if pn.state.curdoc: pn.state.curdoc.on_session_destroyed(analytics_tracker.handle_session_destroyed) + pn.state.curdoc.on_session_destroyed(_release_dead_session_views) app_layout = pn.Column(analytics_tracker.cookie_bridge, layout, margin=0) app_layout._analytics_tracker = analytics_tracker diff --git a/tests/test_session_leak.py b/tests/test_session_leak.py new file mode 100644 index 0000000..d8e48c6 --- /dev/null +++ b/tests/test_session_leak.py @@ -0,0 +1,203 @@ +"""Regression tests: a closed browser tab must not keep its session in memory. + +Production RSS grew without bound (~3.4 GB) and the container was OOM-killed. +A load test against the production image showed every closed session leaving +its whole ``ChannelmapGUI`` (plus ~90k ``Electrode`` objects of wiring maps) +alive. The anchor was ``panel.state._views``: Panel registers the per-session +``NotificationArea`` there, and its teardown hook is only reached through +``state._notifications[doc]`` -- which Panel's own ``Document.destroy`` +override deletes *before* Bokeh runs the session-destroyed callbacks. So the +view entry, and through it the dead Document and everything on it, is never +released (Panel 1.7.5 through at least 1.9.4). +""" + +from __future__ import annotations + +import gc +import socket +import time +import types + +import panel as pn +import pytest +from bokeh.client import pull_session +from bokeh.document import Document +from panel.io.state import state + +from pixelmap.gui import gui as gui_module +from pixelmap.gui.gui import _release_dead_session_views + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_for(cond, timeout: float, what: str) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if cond(): + return + time.sleep(0.1) + raise AssertionError(f"timed out waiting for {what}") + + +def _live_guis() -> int: + gc.collect() + return sum(isinstance(o, gui_module.ChannelmapGUI) for o in gc.get_objects()) + + +def _dead_doc_views() -> list: + """``state._views`` entries whose Document has already been destroyed.""" + return [ref for ref, (_v, _r, doc, _c) in state._views.items() + if doc is not None and not hasattr(doc, "_roots")] + + +# -------------------------------------------------------------------------- +# End to end: real Panel server, real session, real teardown +# -------------------------------------------------------------------------- + +@pytest.fixture +def server(): + port = _free_port() + srv = pn.serve( + {"app": gui_module.create_app}, + port=port, + address="127.0.0.1", + show=False, + threaded=True, + check_unused_sessions_milliseconds=100, + unused_session_lifetime_milliseconds=100, + allow_websocket_origin=[f"127.0.0.1:{port}"], + ) + _wait_for(lambda: srv.server_id in state._servers, 30, "server start") + yield f"http://127.0.0.1:{port}/app", state._servers[srv.server_id][0] + srv.stop() + + +def _sessions(bokeh_server): + return [s for ctx in bokeh_server._tornado._applications.values() + for s in ctx.sessions] + + +@pytest.mark.parametrize("n_sessions", [1, 3]) +def test_closed_sessions_release_their_gui(server, n_sessions): + url, bokeh_server = server + baseline = _live_guis() + + for _ in range(n_sessions): + session = pull_session(url=url) + session.close() + + _wait_for(lambda: not _sessions(bokeh_server), 30, "server sessions to be discarded") + # Teardown runs on the server's loop thread; give it a moment to finish. + _wait_for(lambda: _live_guis() <= baseline, 15, "dead sessions' GUIs to be collected") + + assert _dead_doc_views() == [] + + +# -------------------------------------------------------------------------- +# The hook itself +# -------------------------------------------------------------------------- + +class _FakeView: + def __init__(self): + self.cleaned = [] + + def _cleanup(self, root): + self.cleaned.append(root) + + +class _ExplodingView: + def _cleanup(self, root): + raise RuntimeError("boom") + + +@pytest.fixture +def views(monkeypatch): + monkeypatch.setattr(state, "_views", {}) + return state._views + + +def _ctx(doc): + return types.SimpleNamespace(_document=doc, id="test-session") + + +def test_hook_drops_only_the_dead_documents_views(views): + dead, alive = Document(), Document() + v1, v2, v3 = _FakeView(), _FakeView(), _FakeView() + views["a"] = (v1, "root-a", dead, None) + views["b"] = (v2, "root-b", dead, None) + views["c"] = (v3, "root-c", alive, None) + + _release_dead_session_views(_ctx(dead)) + + assert set(views) == {"c"} + assert v1.cleaned == ["root-a"] and v2.cleaned == ["root-b"] + assert v3.cleaned == [] + + +def test_hook_with_no_views_for_doc_is_noop(views): + other = Document() + views["c"] = (_FakeView(), "root-c", other, None) + _release_dead_session_views(_ctx(Document())) + assert set(views) == {"c"} + + +def test_hook_with_empty_views(views): + _release_dead_session_views(_ctx(Document())) + assert views == {} + + +@pytest.mark.parametrize("ctx", [None, types.SimpleNamespace(_document=None, id="x")]) +def test_hook_tolerates_missing_document(views, ctx): + views["c"] = (_FakeView(), "root-c", Document(), None) + _release_dead_session_views(ctx) + assert set(views) == {"c"} + + +def test_hook_still_drops_view_when_cleanup_raises(views): + """A failing ``_cleanup`` must not keep the session pinned.""" + doc = Document() + views["a"] = (_ExplodingView(), "root-a", doc, None) + _release_dead_session_views(_ctx(doc)) + assert views == {} + + +# -------------------------------------------------------------------------- +# Canary: is the Panel bug still there? +# -------------------------------------------------------------------------- + +def _plain_notifications_app(): + pn.extension(notifications=True) + return pn.Column(pn.widgets.Button(name="x")) + + +@pytest.mark.xfail( + strict=True, + reason="Panel bug (1.7.5..1.9.4): NotificationArea views outlive destroyed " + "sessions. When this XPASSes, Panel is fixed and " + "_release_dead_session_views can be removed.", +) +def test_panel_still_leaks_notification_views_without_workaround(): + port = _free_port() + srv = pn.serve( + {"app": _plain_notifications_app}, + port=port, + address="127.0.0.1", + show=False, + threaded=True, + check_unused_sessions_milliseconds=100, + unused_session_lifetime_milliseconds=100, + allow_websocket_origin=[f"127.0.0.1:{port}"], + ) + try: + _wait_for(lambda: srv.server_id in state._servers, 30, "server start") + bokeh_server = state._servers[srv.server_id][0] + pull_session(url=f"http://127.0.0.1:{port}/app").close() + _wait_for(lambda: not _sessions(bokeh_server), 30, "session discard") + time.sleep(0.5) + assert _dead_doc_views() == [] + finally: + srv.stop() From 06137cc47d8a74e190e6f3ff3e57ee597ad6d7cf Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:35:45 -0400 Subject: [PATCH 2/2] fix: pin the image's Python to 3.14 uv downloads the newest CPython when unpinned; with 3.15 released, `uv sync` fails because dracopy 2.0.0 has no cp315 wheel or sdist, so the next tagged image build would fail. Refs #6 Assisted-by: ClaudeCode:claude-opus-5-5 --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index c3c897f..72c6350 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,11 @@ WORKDIR /app ENV UV_COMPILE_BYTECODE=1 ENV UV_LINK_MODE=copy ENV UV_CACHE_DIR=/root/.cache/uv +# Pin the interpreter. Unpinned, uv downloads the newest CPython, so the image +# silently changes Python whenever one is released -- and a release can break +# the build outright (CPython 3.15: dracopy 2.0.0 ships no cp315 wheel and no +# sdist, so `uv sync` fails). Bump deliberately once dependencies support it. +ENV UV_PYTHON=3.14 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/