diff --git a/backend/backend_cli.py b/backend/backend_cli.py index 315b93f..a9a9120 100644 --- a/backend/backend_cli.py +++ b/backend/backend_cli.py @@ -678,11 +678,17 @@ def start_backend(host: str, port: int) -> tuple[uvicorn.Server, threading.Threa SystemExit: The server stopped before it was up (e.g. the port is taken). """ from newswitch.app import create_app, load_app_config + from newswitch.logging_setup import configure_logging - app = create_app(load_app_config()) + config = load_app_config() + configure_logging(config.log_to_terminal, config.log_to_file, config.log_file_path) + app = create_app(config) # No access log: every call of the console would add a line (with --verbose: to its menu). + # log_config=None: uvicorn would otherwise put its own handlers back. server = uvicorn.Server( - uvicorn.Config(app, host=host, port=port, log_level="warning", access_log=False) + uvicorn.Config( + app, host=host, port=port, log_level="warning", access_log=False, log_config=None + ) ) thread = threading.Thread(target=server.run, name="backend", daemon=True) thread.start() diff --git a/backend/base_config.yaml b/backend/base_config.yaml index 36074e4..0302019 100644 --- a/backend/base_config.yaml +++ b/backend/base_config.yaml @@ -4,7 +4,7 @@ # Host and port are not set here. They come from BACKEND_HOST / BACKEND_PORT in the # repository's root .env (deployment via docker is still an open point). -use_virtual_microscope: true +use_virtual_microscope: false # SQLite file of the rekuest agent (state history); relative to the working directory db_path: agent_data.db available_cubes: [cube1, cube2, cube3] @@ -15,8 +15,8 @@ schema_dir: configs/schemas # Device configuration; bare names are resolved against config_dir. # newswitch never writes static_config_path; changes always go into persistent_config_path. -static_config_path: newswitch-config.yaml -persistent_config_path: newswitch-config.persistent.yaml +static_config_path: camTest_config.yaml +persistent_config_path: camTest_config.persistent.yaml # true: read static_config_path; on shutdown persistent_config_path is overwritten # with it plus the runtime changes # false: read (and write back) persistent_config_path; on first start it is created @@ -33,3 +33,12 @@ camera_recording_dir: data/recordings # the window always has one; this writes nothing to disk. Without an X11 display only an # error is logged. Set false on headless machines. camera_debug_window: true + +# Logging of the backend (the frontend logs in the browser). Each output gets records from +# its level up: fatal | error | warn | info | debug, or none for no output at all. +log_to_terminal: none +log_to_file: debug +# Log file, relative to the backend's working directory (backend/ with `just dev`). It is +# appended to and rotated at 10 MB (backend.log.1 ... .3 are kept). Missing, or not openable +# (e.g. its folder does not exist): backend.log is used, with a warning in it. +log_file_path: backend.log diff --git a/backend/docs/DEVELOPER_OVERVIEW.md b/backend/docs/DEVELOPER_OVERVIEW.md index 50f9598..6e983eb 100644 --- a/backend/docs/DEVELOPER_OVERVIEW.md +++ b/backend/docs/DEVELOPER_OVERVIEW.md @@ -58,6 +58,20 @@ A device kind missing from the file, or `config_file=None` (the tests), keeps th built-in virtual defaults. On shutdown exposure, gain, wheel positions and stage position are written back; comments stay intact. Details: `backend/configs/README.md`. +### Logging (`newswitch/logging_setup.py`) + +The backend does not print; every module logs through `logging.getLogger(__name__)` (ruff +rule T20; only the command-line tools `newswitch/cli.py`, `newswitch/config_io/__main__.py` +and `scripts/` print their results). `base_config.yaml` sets one level for the terminal +(`log_to_terminal`) and one for the log file (`log_to_file`): `fatal`, `error`, `warn`, +`info`, `debug` or `none`. The file (`log_file_path`, relative to the working directory) is +appended to and rotated at 10 MB; if it cannot be opened, `backend.log` is used with a +warning. `configure_logging` runs in the entry points (`main.py`, `newswitch.app.main`, +`backend_cli.py`), not in `create_app`, so tests write no log files. It also takes away +uvicorn's own handlers; `route_to_root` and `log_prints` do the same for libraries that +attach handlers or print (uc2rest in `managers/uc2/rest_bus.py`). Not covered: uvicorn's +reloader process under `--reload`, prints of other libraries, native output. + ### Registered functions Everything the frontend can call lives in `newswitch/registers/`: one module per device diff --git a/backend/docs/LIVE-IMAGE-PIPELINE.md b/backend/docs/LIVE-IMAGE-PIPELINE.md index 6897050..55664ba 100644 --- a/backend/docs/LIVE-IMAGE-PIPELINE.md +++ b/backend/docs/LIVE-IMAGE-PIPELINE.md @@ -277,8 +277,8 @@ sequenceDiagram H->>H: accept BR->>H: auth frame with token H->>H: authenticate_stream_websocket, close 1008 if invalid - H->>H: get_broadcaster_from_websocket - Note over H: Fails today, the agent has no get_context_for_type (issue 1) + H->>H: await_broadcaster, context frame_broadcaster from agent.contexts + Note over H: waits up to 10 s for the agent's startup hooks, then closes 1011 H->>FB: get_subscription_(slot, H264EncoderConfig()) FB->>EN: create if no encoder for (slot, config), store in _encoders FB->>EN: astart() if not running @@ -303,7 +303,7 @@ sequenceDiagram end BR--xH: disconnect (LivePlane unmounts, e.g. live view stopped) - Note over H: noticed only when the next send_bytes fails (issue 3) + Note over H: noticed only when the next send_bytes fails (issue 1) H->>FB: arelease_subscription(subscription) FB->>EN: aunsubscribe, astop() if it was the last subscriber EN->>FF: kill @@ -396,17 +396,12 @@ after `onopen` the hook sends the auth frame (`authFrame()`, the token). A close Tracked in `.claude/OpenPoints.md`. -1. **The stream route crashes on every connection (OP #28).** - `get_broadcaster_from_websocket` (`liveview.py:75`) calls `agent.get_context_for_type(...)`, - which rekuest_next 2.3.1 does not have (it offers `agent.contexts[key]` / - `aget_context(key)`; the key is `frame_broadcaster`). The handler raises `AttributeError` - before `get_subscription_`. No encoder is ever created, and no image reaches the frontend. - Reproduced with camTest: 0 encoders and 0 bytes with a connected, authenticated client. - `tests/test_auth.py` only tests the rejected path. -2. **ffmpeg's stderr is never read (OP #29).** `stderr=PIPE` (`h264_encoder.py:118`) without a - reader: ffmpeg writes progress lines continuously, and once the pipe buffer (about 64 KB) is - full, ffmpeg blocks and the stream freezes. -3. **Further risks (OP #30):** +Fixed on 2026-10-27: the stream route crashed on every connection (it called +`agent.get_context_for_type`, which rekuest_next 2.3.1 does not have), and ffmpeg's unread +stderr pipe froze the H.264 stream after a few minutes (ffmpeg now runs with +`-loglevel error -nostats`, stderr discarded). + +1. **Further risks (OP #30):** - Thread safety. `encode_frame` runs on the worker thread: H.264 calls `StreamWriter.write` there, Zstd calls `asyncio.Queue.put_nowait`. Neither is thread-safe. - Dropped H.264 chunks. The queue keeps one 4096-byte chunk and drops the oldest. Dropping @@ -423,7 +418,7 @@ Tracked in `.claude/OpenPoints.md`. `uint8` full frames (unused today). - Dead frontend code. `components/microscope/LiveView.tsx` is not mounted anywhere; its `StreamingView` connects to `VITE_WEBSOCKET_URL + "/video"`, a route that does not exist. -4. **Frame rate limited by `koil.sleep` (OP #3).** The live loop's 10 ms pause takes about +2. **Frame rate limited by `koil.sleep` (OP #3).** The live loop's 10 ms pause takes about 140 ms, which limits the stream to about 7 fps. --- @@ -442,4 +437,5 @@ Tracked in `.claude/OpenPoints.md`. `--verbose` is given. - **Browser side:** DevTools → Network → WS: is there a connection to `/stream/h264/`, on the right port, and with which close code? 1006 (no close frame) points to a - server-side exception, 1008 to a rejected token. + server-side exception, 1008 to a rejected token, 1011 to a FrameBroadcaster the agent did + not provide within 10 s (startup hooks failed or still running). diff --git a/backend/main.py b/backend/main.py index 8abcb47..2842188 100644 --- a/backend/main.py +++ b/backend/main.py @@ -16,20 +16,29 @@ Run directly (``python main.py``, e.g. in the debugger) the app serves on BACKEND_HOST / BACKEND_PORT from the environment, like `newswitch.app.main`. + +Logging follows log_to_terminal / log_to_file / log_file_path of the settings. Under +``uvicorn main:app`` uvicorn sets up its own logging first and imports this module afterwards, +so the settings win. """ import os from newswitch.app import create_app, load_app_config +from newswitch.logging_setup import configure_logging -app = create_app(load_app_config()) +config = load_app_config() +configure_logging(config.log_to_terminal, config.log_to_file, config.log_file_path) +app = create_app(config) if __name__ == "__main__": import uvicorn + # log_config=None: uvicorn would otherwise put its own handlers back uvicorn.run( app, host=os.environ.get("BACKEND_HOST", "0.0.0.0"), port=int(os.environ.get("BACKEND_PORT", "8099")), + log_config=None, ) diff --git a/backend/newswitch/app.py b/backend/newswitch/app.py index 27165e6..5270dec 100644 --- a/backend/newswitch/app.py +++ b/backend/newswitch/app.py @@ -56,6 +56,7 @@ from newswitch.protocols.objective import ObjectiveLens from newswitch.protocols.uc2 import UC2BusManager, UC2State from newswitch import registers # noqa: F401 (registration side effects: every action) +from newswitch.logging_setup import LogLevel, configure_logging from newswitch.managers.expanse_manager import ExpanseManager from newswitch.managers.metadata_manager import MetadataManager from newswitch.protocols import ( @@ -163,6 +164,13 @@ class ImswitchConfig(BaseSettings): """Show every camera's newest image in a window of uc2_cam_py (debugging without the frontend); the cameras then acquire from startup on.""" + log_to_terminal: LogLevel = "none" + """Lowest level of the log records written to the terminal (see `logging_setup`).""" + log_to_file: LogLevel = "debug" + """Lowest level of the log records written to `log_file_path`.""" + log_file_path: Optional[str] = "backend.log" + """Log file, relative to the working directory. Missing or not openable: backend.log.""" + # UC2 hardware transport (used when use_virtual_microscope is False). # "canopen" (preferred, multi-node CAN bus) or "rest" (single ESP32 master # over serial JSON). Both are plug-in replacements behind UC2BusManager. @@ -361,21 +369,21 @@ async def provide_managers( # TODO: can we make this adaptive so that we read t app_context = ImswitchConfig() mode = "virtual" if app_context.use_virtual_microscope else "real hardware" - print(f"Initializing {mode} microscope managers") - print(f"UC2 transport: {app_context.uc2_transport}") + logger.info("Initializing %s microscope managers", mode) + logger.info("UC2 transport: %s", app_context.uc2_transport) if app_context.uc2_transport == "canopen": - print( - "CAN interface: " - f"{app_context.uc2_can_interface}, " - f"channel={app_context.uc2_can_channel}, " - f"port={app_context.uc2_can_port}" + logger.info( + "CAN interface: %s, channel=%s, port=%s", + app_context.uc2_can_interface, + app_context.uc2_can_channel, + app_context.uc2_can_port, ) frame_broadcaster = FrameBroadcaster() config = app_context if config.use_virtual_microscope: - print("Using virtual microscope managers.") + logger.info("Using virtual microscope managers.") illumination_state = IlluminationState() @@ -529,7 +537,7 @@ async def provide_managers( # TODO: can we make this adaptive so that we read t hook_manager.register_hook(protocols.SoftwareAutofocusHook, software_autofocus_hook) hook_manager.register_hook( protocols.ZCalibrationHook, - lambda hook, context: print("Z-calibration hook executed"), + lambda hook, context: logger.info("Z-calibration hook executed"), ) expanse_state = protocols.ExpanseState() @@ -626,9 +634,9 @@ def save_device_config( adapters.sync_runtime_state( cfg_file.config, camera_state, objective_state, filter_bank_state, stage_state ) - print(f"Saved device configuration to {cfg_file.save()}") + logger.info("Saved device configuration to %s", cfg_file.save()) except config_io.ConfigError as exc: - print(f"Device configuration not saved: {exc}") + logger.warning("Device configuration not saved: %s", exc) @shutdown @@ -758,11 +766,14 @@ def main() -> None: settings come from `load_app_config`, so ``NEWSWITCH_CONFIG`` applies as for main.py. """ config = load_app_config() + configure_logging(config.log_to_terminal, config.log_to_file, config.log_file_path) app = create_app(config) + # log_config=None: uvicorn would otherwise put its own handlers back uvicorn.run( app, host=os.environ.get("BACKEND_HOST", "0.0.0.0"), port=int(os.environ.get("BACKEND_PORT", "8099")), + log_config=None, ) diff --git a/backend/newswitch/broadcasters/encoders/h264_encoder.py b/backend/newswitch/broadcasters/encoders/h264_encoder.py index 3146e62..8752c82 100644 --- a/backend/newswitch/broadcasters/encoders/h264_encoder.py +++ b/backend/newswitch/broadcasters/encoders/h264_encoder.py @@ -71,8 +71,14 @@ async def astart(self) -> None: if self._running: return + # Nobody reads ffmpeg's stderr, and a full pipe would block ffmpeg and freeze the + # stream: keep it quiet (no banner, no progress line) and discard what is left. command = [ "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-nostats", "-y", "-f", "rawvideo", @@ -115,7 +121,7 @@ async def astart(self) -> None: *command, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, ) self._running = True self._reader_task = asyncio.create_task(self._read_output()) diff --git a/backend/newswitch/broadcasters/frame.py b/backend/newswitch/broadcasters/frame.py index 28794f2..52e46a5 100644 --- a/backend/newswitch/broadcasters/frame.py +++ b/backend/newswitch/broadcasters/frame.py @@ -9,6 +9,7 @@ - Shared encoders for clients with matching configurations """ +import logging from typing import Optional import asyncio import threading @@ -23,6 +24,8 @@ from newswitch.broadcasters.encoders.config import EncoderConfig from newswitch.broadcasters.encoders.base import EncoderSubscriptionProtocol +logger = logging.getLogger(__name__) + @context class FrameBroadcaster: @@ -136,6 +139,17 @@ async def arelease_subscription(self, subscription: EncoderSubscriptionProtocol) def _schedule_encoder_frames(self, detector_slot: int, frame: np.ndarray) -> None: """Schedule frame encoding for active encoders of one detector slot.""" + if logger.isEnabledFor(logging.DEBUG): # max/min of every frame are not free + logger.debug( + "Scheduling frame for detector slot %s with shape %s, dtype %s, max %s, " + "min %s, encoders: %d", + detector_slot, + frame.shape, + frame.dtype, + frame.max(), + frame.min(), + len(self._encoders), + ) for (slot, _), encoder in self._encoders.items(): if slot != detector_slot: continue diff --git a/backend/newswitch/hooks/software_autofocus.py b/backend/newswitch/hooks/software_autofocus.py index a70312a..c04c87f 100644 --- a/backend/newswitch/hooks/software_autofocus.py +++ b/backend/newswitch/hooks/software_autofocus.py @@ -1,8 +1,11 @@ """Software autofocus hook: refocus using the shared autofocus sweep core.""" +import logging from newswitch import protocols from newswitch.routines.autofocus import autofocus_sweep +logger = logging.getLogger(__name__) + DEFAULT_Z_RANGE = 20.0 # micrometers, centered on the current Z DEFAULT_STEPS = 11 @@ -17,4 +20,4 @@ def software_autofocus_hook(hook: protocols.Hook, context: protocols.HookContext ] best_z, metrics = autofocus_sweep(stage, context.detector_manager, z_positions) stage.move(z=best_z, is_absolute=True) - print(f"Autofocus hook: best z={best_z:.2f} (metric {max(metrics):.1f})") + logger.info("Autofocus hook: best z=%.2f (metric %.1f)", best_z, max(metrics)) diff --git a/backend/newswitch/logging_setup.py b/backend/newswitch/logging_setup.py new file mode 100644 index 0000000..d9682fc --- /dev/null +++ b/backend/newswitch/logging_setup.py @@ -0,0 +1,186 @@ +"""Logging of the backend: one level for the terminal, one for the log file. + +The levels and the file come from base_config.yaml (``log_to_terminal``, ``log_to_file``, +``log_file_path``). `configure_logging` is called by the entry points (``main.py``, +`newswitch.app.main`, ``backend_cli.py``), not by `create_app`, so tests write no log files. + +Libraries that attach handlers of their own bypass these levels; `route_to_root` takes their +handlers away. `log_prints` turns what a library prints into log records. +""" + +import contextlib +import io +import logging +import logging.handlers +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Literal + +logger = logging.getLogger(__name__) + +#: Values of ``log_to_terminal`` and ``log_to_file``: records from this level up are written; +#: ``none`` writes nothing. +LogLevel = Literal["fatal", "error", "warn", "info", "debug", "none"] + +LOG_LEVELS: dict[str, int | None] = { + "fatal": logging.CRITICAL, + "error": logging.ERROR, + "warn": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG, + "none": None, +} + +#: Log file used when ``log_file_path`` is missing or cannot be opened; relative to the +#: working directory. +DEFAULT_LOG_FILE = "backend.log" +#: The log file is rotated at this size; this many old ones are kept (backend.log.1, ...). +MAX_LOG_FILE_BYTES = 10 * 1024 * 1024 +LOG_FILE_BACKUPS = 3 + +LOG_FORMAT = "%(asctime)s %(levelname)-8s %(name)s: %(message)s" + +#: uvicorn attaches handlers of its own to these loggers. +UVICORN_LOGGERS = ("uvicorn", "uvicorn.error", "uvicorn.access") + +#: Handlers installed by `configure_logging`; a second call replaces them. +_installed: list[logging.Handler] = [] + + +def configure_logging(terminal: LogLevel, file: LogLevel, file_path: str | None) -> None: + """Write log records to stderr and to a rotating log file, each from its own level up. + + Calling it again replaces the handlers of the previous call; handlers installed by + others on the root logger stay. uvicorn's loggers lose their own handlers, so their + records follow these levels too. Warnings of the `warnings` module become log records. + + Args: + terminal: Lowest level written to stderr. + file: Lowest level written to the log file. + file_path: The log file, relative to the working directory. If it is missing or + cannot be opened for appending, `DEFAULT_LOG_FILE` is used and a warning logged. + """ + root = logging.getLogger() + for handler in _installed: + root.removeHandler(handler) + handler.close() + _installed.clear() + + terminal_level = LOG_LEVELS[terminal] + file_level = LOG_LEVELS[file] + + if terminal_level is not None: + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setLevel(terminal_level) + _installed.append(stream_handler) + + # Logged only once the handlers are in place, so they reach the terminal and the file + problems: list[tuple[int, str]] = [] + if file_level is not None: + file_handler = _open_log_file(file_path, problems) + if file_handler is not None: + file_handler.setLevel(file_level) + _installed.append(file_handler) + + levels = [handler.level for handler in _installed] + if not _installed: + # Without any handler, Python's last resort would still print warnings to stderr + _installed.append(logging.NullHandler()) + + formatter = logging.Formatter(LOG_FORMAT) + for handler in _installed: + handler.setFormatter(formatter) + root.addHandler(handler) + root.setLevel(min(levels) if levels else logging.CRITICAL + 1) + + route_to_root(*UVICORN_LOGGERS) + for name in UVICORN_LOGGERS: + logging.getLogger(name).setLevel(logging.NOTSET) + logging.captureWarnings(True) + + for level, message in problems: + logger.log(level, "%s", message) + + +def _open_log_file( + file_path: str | None, problems: list[tuple[int, str]] +) -> logging.Handler | None: + """Open `file_path` for appending, falling back to `DEFAULT_LOG_FILE`. + + Args: + file_path: The configured log file. + problems: Receives a (level, message) for every fallback taken. + + Returns: + The handler, or None if not even `DEFAULT_LOG_FILE` can be opened. + """ + fallback = Path(DEFAULT_LOG_FILE).absolute() + if file_path and file_path.strip(): + try: + return _rotating_file_handler(Path(file_path)) + except OSError as exc: + problems.append( + ( + logging.WARNING, + f"log_file_path {file_path!r} cannot be used ({exc}); logging to {fallback}", + ) + ) + else: + problems.append((logging.WARNING, f"no log_file_path given; logging to {fallback}")) + try: + return _rotating_file_handler(fallback) + except OSError as exc: + problems.append((logging.ERROR, f"cannot open {fallback} ({exc}); no log file is written")) + return None + + +def _rotating_file_handler(path: Path) -> logging.Handler: + """A handler appending to `path`, rotated at `MAX_LOG_FILE_BYTES`. + + Raises: + OSError: The file cannot be opened (missing folder, a folder, no permission). + """ + return logging.handlers.RotatingFileHandler( + path, maxBytes=MAX_LOG_FILE_BYTES, backupCount=LOG_FILE_BACKUPS, encoding="utf-8" + ) + + +def route_to_root(*names: str) -> None: + """Take away the handlers of these loggers, so their records go through the root logger. + + For libraries that attach handlers of their own (uvicorn, uc2rest), which would write + regardless of the levels set by `configure_logging`. + """ + for name in names: + library_logger = logging.getLogger(name) + for handler in list(library_logger.handlers): + library_logger.removeHandler(handler) + library_logger.propagate = True + + +class _PrintsToLog(io.TextIOBase): + """Text stream that logs every non-empty line written to it.""" + + def __init__(self, target: logging.Logger, level: int) -> None: + """Log to `target` at `level`.""" + self._target = target + self._level = level + + def write(self, text: str) -> int: + """Log each non-empty line of `text`.""" + for line in text.splitlines(): + if line.strip(): + self._target.log(self._level, "%s", line) + return len(text) + + +@contextlib.contextmanager +def log_prints(target: logging.Logger, level: int) -> Iterator[None]: + """Log what is printed inside the block instead of writing it to stdout. + + `sys.stdout` is replaced for the whole process while the block runs, so prints of other + threads land in the log as well. + """ + with contextlib.redirect_stdout(_PrintsToLog(target, level)): + yield diff --git a/backend/newswitch/managers/acquistion_manager.py b/backend/newswitch/managers/acquistion_manager.py index 97c499c..d051630 100644 --- a/backend/newswitch/managers/acquistion_manager.py +++ b/backend/newswitch/managers/acquistion_manager.py @@ -3,6 +3,7 @@ from __future__ import annotations +import logging from newswitch.protocols import MetadataManager from newswitch.protocols.core import ArrayMetadata, Frame, Scale from newswitch.protocols.expanse import ExpanseManager @@ -10,6 +11,8 @@ from newswitch.protocols.light_path import LightPathManager from newswitch.protocols.cache import CacheManager +logger = logging.getLogger(__name__) + class AcquistionManager: """Registry and execution for acquisition hooks.""" @@ -41,8 +44,11 @@ def acquire(self, save_to_disk: bool = False) -> list[Frame]: for detector in self.detector_manager.state.detectors: # Simulate image acquisition from the detector - print( - f"Acquiring image from detector slot {detector.slot} with exposure time {detector.current_exposure_time} ms {detector.is_active}" + logger.debug( + "Acquiring image from detector slot %s with exposure time %s s, active: %s", + detector.slot, + detector.current_exposure_time, + detector.is_active, ) if detector.is_active: image_data = self.detector_manager.capture_image( @@ -75,7 +81,7 @@ def acquire(self, save_to_disk: bool = False) -> list[Frame]: ), ) - print("Added frame to expanse state with ID:", frame.id) + logger.debug("Added frame to expanse state with ID: %s", frame.id) self.expanse_manager.add_frame(frame) frames.append(frame) diff --git a/backend/newswitch/managers/expanse_manager.py b/backend/newswitch/managers/expanse_manager.py index c6b4c04..6826ed2 100644 --- a/backend/newswitch/managers/expanse_manager.py +++ b/backend/newswitch/managers/expanse_manager.py @@ -3,8 +3,11 @@ from __future__ import annotations +import logging from newswitch.protocols.expanse import ExpanseState, Image, Frame +logger = logging.getLogger(__name__) + class ExpanseManager: """Registry and execution for acquisition hooks.""" @@ -20,7 +23,7 @@ def add_image(self, image: Image) -> None: def add_frame(self, frame: Frame) -> None: """Add a new frame to the current state.""" self.state.current_frames.append(frame) - print(self.state) + logger.debug("%s", self.state) def reset_expanse(self) -> None: """Reset the expanse state, clearing all current images.""" diff --git a/backend/newswitch/managers/python_hook_manager.py b/backend/newswitch/managers/python_hook_manager.py index 60e5d64..3e64960 100644 --- a/backend/newswitch/managers/python_hook_manager.py +++ b/backend/newswitch/managers/python_hook_manager.py @@ -2,10 +2,13 @@ from __future__ import annotations +import logging from typing import Dict, Type from newswitch.protocols.hook_manager import HookContext, Hook, HookHandler +logger = logging.getLogger(__name__) + class PythonHookManager: """Registry and execution for acquisition hooks.""" @@ -36,4 +39,4 @@ def execute_all(self, hooks: list[Hook]) -> None: """Execute a list of hooks in order.""" for hook in hooks: # self.execute(hook) - print(f"Executing hook: {hook}") + logger.info("Executing hook: %s", hook) diff --git a/backend/newswitch/managers/uc2/rest_bus.py b/backend/newswitch/managers/uc2/rest_bus.py index 76d9a21..b16ef0f 100644 --- a/backend/newswitch/managers/uc2/rest_bus.py +++ b/backend/newswitch/managers/uc2/rest_bus.py @@ -18,6 +18,7 @@ from rekuest_next import model +from newswitch.logging_setup import log_prints, route_to_root from newswitch.managers.uc2.event_broker import UC2EventBroker from newswitch.protocols.uc2 import ( BusError, @@ -146,13 +147,20 @@ async def _connect_and_pump(self) -> None: ) from exc self._broker.bind_loop() - client = await AsyncUC2Client.create( - serialport=self.config.serialport, - baudrate=self.config.baudrate, - identity=self.config.identity, - device_id=self.config.device_id, - requireMaster=self.config.require_master, - ) + # uc2rest prints while it builds the client, and attaches a DEBUG handler of its own + # to its logger every time (its records would bypass our levels and repeat). What it + # logs before the handler is taken away still reaches the terminal. + try: + with log_prints(logger, logging.DEBUG): + client = await AsyncUC2Client.create( + serialport=self.config.serialport, + baudrate=self.config.baudrate, + identity=self.config.identity, + device_id=self.config.device_id, + requireMaster=self.config.require_master, + ) + finally: + route_to_root("uc2rest.logger", "uc2rest.mserial") if not client.is_connected: # uc2rest degrades to a MockSerial dummy when no board answers; # treat that as a connection failure so the backoff loop keeps diff --git a/backend/newswitch/registers/debug.py b/backend/newswitch/registers/debug.py index 701ad00..7e495c4 100644 --- a/backend/newswitch/registers/debug.py +++ b/backend/newswitch/registers/debug.py @@ -4,6 +4,7 @@ action that must release its locks, a critical failure, a function that never ends). """ +import logging import koil from rekuest_next import pausepoint, progress from rekuest_next.register import register @@ -18,6 +19,8 @@ StageState, ) +logger = logging.getLogger(__name__) + @register def long_stuff_running() -> None: @@ -56,8 +59,7 @@ def dump_states_to_stdin( expanse_state: protocols.ExpanseState, ) -> None: """Dump the current states of all managers for debugging purposes.""" - print("Dumping states to JSON:") - print(len(expanse_state.current_frames)) + logger.info("Dumping states to JSON: %d current frames", len(expanse_state.current_frames)) @register diff --git a/backend/newswitch/registers/detector.py b/backend/newswitch/registers/detector.py index ed0293c..7a22973 100644 --- a/backend/newswitch/registers/detector.py +++ b/backend/newswitch/registers/detector.py @@ -1,5 +1,6 @@ """Registered functions for the detectors (cameras). Importing this module registers them.""" +import logging from rekuest_next.register import register from newswitch import protocols @@ -7,6 +8,8 @@ from newswitch.protocols.core import Frame from newswitch.protocols.detector import Detector +logger = logging.getLogger(__name__) + @register def capture_image( @@ -52,7 +55,7 @@ def start_live_view(detector: DetectorManager) -> str: Call this to begin streaming frames captured by the detector to connected video clients. """ - print("Starting live view broadcasting...") + logger.info("Starting live view broadcasting...") detector.start_acquire_live(save_to_disk=False) return "Broadcasting started. Frames will now be sent to video clients." @@ -66,7 +69,7 @@ def start_live_view_and_save_to_disk(detector: DetectorManager) -> str: stop_live_view or until the detector is deactivated. The virtual detector cannot record and fails. """ - print("Starting live view broadcasting with saving to disk...") + logger.info("Starting live view broadcasting with saving to disk...") detector.start_acquire_live(save_to_disk=True) return "Broadcasting started. Frames are sent to video clients and written to disk." @@ -79,7 +82,7 @@ def stop_live_view(detector: DetectorManager) -> str: Call this to stop streaming frames captured by the detector to connected video clients. Writing to disk ends with it. """ - print("Stopping live view broadcasting...") + logger.info("Stopping live view broadcasting...") detector.stop_acquire_live() return "Broadcasting stopped. No frames will be sent to video clients." @@ -92,7 +95,7 @@ def activate_detector(detector: DetectorManager, slot: int) -> None: Args: slot: Detector slot number to activate """ - print(f"Activating detector in slot {slot}...") + logger.info("Activating detector in slot %s...", slot) detector.activate_detector(slot) return None @@ -105,7 +108,7 @@ def deactivate_detector(detector: DetectorManager, slot: int) -> str: Args: slot: Detector slot number to deactivate """ - print(f"Deactivating detector in slot {slot}...") + logger.info("Deactivating detector in slot %s...", slot) detector.deactivate_detector(slot) return f"Detector slot {slot} deactivated" diff --git a/backend/newswitch/routes/ws/liveview.py b/backend/newswitch/routes/ws/liveview.py index 204d493..ff9c318 100644 --- a/backend/newswitch/routes/ws/liveview.py +++ b/backend/newswitch/routes/ws/liveview.py @@ -2,11 +2,15 @@ WebSocket routes for live video streaming. """ +import logging from fastapi import APIRouter, WebSocket, WebSocketDisconnect import asyncio +from rekuest_next.agents.context import get_context_name from newswitch.auth import Authenticator from newswitch.broadcasters import FrameBroadcaster, ZstdEncoderConfig, H264EncoderConfig +logger = logging.getLogger(__name__) + router = APIRouter() @@ -14,6 +18,12 @@ #: because the frame is sent from the client's `onopen` handler. AUTH_FRAME_TIMEOUT = 10.0 +#: How long a stream waits for the agent to provide the FrameBroadcaster. The agent's +#: startup hooks run in a task started after the app began serving, so a client that +#: connects right after startup can arrive before the context exists - and the frontend +#: does not reconnect when the socket is closed. +BROADCASTER_WAIT_TIMEOUT = 10.0 + async def authenticate_stream_websocket(websocket: WebSocket) -> bool: """Consume and validate the client's opening auth frame. @@ -59,26 +69,43 @@ async def _close_unauthorized(websocket: WebSocket) -> None: pass -# This is a helper function to get the broadcaster from the app state via WebSocket. -# By default, the app agent, which is set on app.state.agent holds references -# to all contexts, including the FrameBroadcaster context which manages video streaming. -def get_broadcaster_from_websocket(websocket: WebSocket) -> FrameBroadcaster: - """ - Get the FrameBroadcaster from the app state via WebSocket. +async def await_broadcaster(websocket: WebSocket) -> FrameBroadcaster | None: + """Get the FrameBroadcaster from the agent's contexts, waiting until it is provided. - The broadcaster is set on app.state.agent context during startup. - """ - agent = getattr(websocket.app.state, "agent", None) - if agent is None: - raise RuntimeError("Agent not available") + The agent (`app.state.agent`) keeps its contexts in a dict keyed by context name; + `get_context_name` gives the same key rekuest assigns when the startup hook returns + the broadcaster. - broadcaster = agent.get_context_for_type(FrameBroadcaster) - if broadcaster is None: - raise RuntimeError("FrameBroadcaster not available") + Returns: + The broadcaster, or None if it did not appear within `BROADCASTER_WAIT_TIMEOUT`. + Raises: + TypeError: The context under the broadcaster's name is something else. + """ + contexts = websocket.app.state.agent.contexts + key = get_context_name(FrameBroadcaster) + deadline = asyncio.get_running_loop().time() + BROADCASTER_WAIT_TIMEOUT + + while (broadcaster := contexts.get(key)) is None: + if asyncio.get_running_loop().time() > deadline: + return None + await asyncio.sleep(0.1) + + if not isinstance(broadcaster, FrameBroadcaster): + raise TypeError( + f"Context {key!r} is a {type(broadcaster).__name__}, not a FrameBroadcaster" + ) return broadcaster +async def _close_unavailable(websocket: WebSocket) -> None: + """Close with 1011, tolerating a client that already went away.""" + try: + await websocket.close(code=1011, reason="frame broadcaster unavailable") + except RuntimeError: + pass + + @router.websocket("/zstd/{detector_slot}") async def stream_zstd(websocket: WebSocket, detector_slot: int) -> None: """ @@ -92,7 +119,10 @@ async def stream_zstd(websocket: WebSocket, detector_slot: int) -> None: if not await authenticate_stream_websocket(websocket): return - broadcaster = get_broadcaster_from_websocket(websocket) + broadcaster = await await_broadcaster(websocket) + if broadcaster is None: + await _close_unavailable(websocket) + return # Get subscription to shared encoder config = ZstdEncoderConfig(level=1, threshold=0.0, use_delta=True) @@ -118,8 +148,8 @@ async def stream_zstd(websocket: WebSocket, detector_slot: int) -> None: except WebSocketDisconnect: pass - except Exception as e: - print(f"Stream error: {e}") + except Exception: + logger.exception("Stream error") finally: await broadcaster.arelease_subscription(subscription) @@ -137,7 +167,10 @@ async def stream_h264(websocket: WebSocket, detector_slot: int) -> None: if not await authenticate_stream_websocket(websocket): return - broadcaster = get_broadcaster_from_websocket(websocket) + broadcaster = await await_broadcaster(websocket) + if broadcaster is None: + await _close_unavailable(websocket) + return # Get subscription to shared encoder config = H264EncoderConfig() @@ -163,7 +196,7 @@ async def stream_h264(websocket: WebSocket, detector_slot: int) -> None: except WebSocketDisconnect: pass - except Exception as e: - print(f"Stream error: {e}") + except Exception: + logger.exception("Stream error") finally: await broadcaster.arelease_subscription(subscription) diff --git a/backend/newswitch/routines/calibration.py b/backend/newswitch/routines/calibration.py index bf4cb22..a2bf9db 100644 --- a/backend/newswitch/routines/calibration.py +++ b/backend/newswitch/routines/calibration.py @@ -1,5 +1,6 @@ """Routine for calibrating the affine transformation between stage coordinates and image coordinates.""" +import logging import copy import numpy as np from newswitch import protocols @@ -7,6 +8,8 @@ from newswitch.protocols.calibration import CalibratedLightPath from newswitch.protocols.core import AffineMatrix +logger = logging.getLogger(__name__) + def calibrate_light_path( light_path_manager: protocols.LightPathManager, @@ -48,7 +51,7 @@ def calibrate_light_path( images.append(frame) affine_matrix_x = calculate_3d_affine(images=images, stage_positions=stage_states) - print(f"Calculated affine matrix: {affine_matrix_x.matrix}") + logger.info("Calculated affine matrix: %s", affine_matrix_x.matrix) # Simulate the calculation of the affine transformation matrix based on the captured image affine_matrix: AffineMatrix = [ [0.34, 0.0, 0.0, 0], diff --git a/backend/newswitch/routines/multidimensional_acquisition.py b/backend/newswitch/routines/multidimensional_acquisition.py index 3d0941c..52dd24f 100644 --- a/backend/newswitch/routines/multidimensional_acquisition.py +++ b/backend/newswitch/routines/multidimensional_acquisition.py @@ -1,5 +1,6 @@ "Routine for performing a multidimensional acquisition with multiple timepoints, positions, z-stacks, and channels." +import logging from datetime import datetime import random from enum import Enum @@ -14,6 +15,8 @@ import time from rekuest_next import pausepoint +logger = logging.getLogger(__name__) + class PositionOrder(str, Enum): """Defines the order in which stage positions are visited during acquisition.""" @@ -153,11 +156,11 @@ def acquire_multidimensional_acquisition( list[AcquiredImage]: List of acquired images with metadata. """ acquired: list[Frame] = [] - print(f"Starting acquisition with config: {config}") + logger.info("Starting acquisition with config: %s", config) hook_manager.execute_all(config.m_hooks or []) - print(f"Executing acquisition hooks: {config.m_hooks}") + logger.info("Executing acquisition hooks: %s", config.m_hooks) # Calculate total number of images to acquire for progress tracking total_images = 0 @@ -184,8 +187,11 @@ def acquire_multidimensional_acquisition( if timepoint.position_order == PositionOrder.RANDOM: positions = random.sample(positions, k=len(positions)) - print( - f"Acquiring timepoint {time_index} with positions: {positions} and hooks: {timepoint.t_hooks}" + logger.info( + "Acquiring timepoint %s with positions: %s and hooks: %s", + time_index, + positions, + timepoint.t_hooks, ) for position_index, position in enumerate(positions): @@ -238,7 +244,7 @@ def acquire_multidimensional_acquisition( f"Acquired image {current_image}/{total_images} (T{time_index + 1}, P{position_index + 1}, Z{z_index + 1}, Stream{stream_index + 1})", ) - print("streaming acquired image data...") + logger.debug("streaming acquired image data...") for channel in channels_to_disable: illumination_manager.turn_off_channel(channel) diff --git a/backend/newswitch/routines/region_scan.py b/backend/newswitch/routines/region_scan.py index 1fc583d..d96c31a 100644 --- a/backend/newswitch/routines/region_scan.py +++ b/backend/newswitch/routines/region_scan.py @@ -1,5 +1,6 @@ """Routine for scanning a rectangular region as a tiled grid of stage positions.""" +import logging import math from enum import Enum from typing import Annotated @@ -9,6 +10,8 @@ from newswitch import protocols from newswitch.protocols.core import Image +logger = logging.getLogger(__name__) + class ScanOrder(str, Enum): """Defines the order in which stage positions are visited during acquisition.""" @@ -48,8 +51,13 @@ def scan_region( list[AcquiredImage]: List of acquired images with metadata. """ - print( - f"Scanning region from ({start_x}, {start_y}) to ({end_x}, {end_y}) in {scan_order} order." + logger.info( + "Scanning region from (%s, %s) to (%s, %s) in %s order.", + start_x, + start_y, + end_x, + end_y, + scan_order, ) # 1. Calculate FOV for the current objective and detector configuration @@ -73,8 +81,10 @@ def scan_region( metadata.affine_matrix[1][1] * current_detector.height ) # Placeholder: e.g., pixel_size_y * sensor_height # Placeholder: e.g., pixel_size_y * sensor_height - print( - f"Calculated FOV: {fov_width:.2f} x {fov_height:.2f} (um) based on current configuration." + logger.info( + "Calculated FOV: %.2f x %.2f (um) based on current configuration.", + fov_width, + fov_height, ) # 2. Define Region Bounds, ensuring min/max are correctly assigned regardless of drag direction @@ -124,8 +134,11 @@ def scan_region( y = start_grid_y + actual_r * step_y path_points.append((x, y)) - print( - f"Calculated {cols} cols and {rows} rows ({len(path_points)} total points) based on FOV and overlap." + logger.info( + "Calculated %d cols and %d rows (%d total points) based on FOV and overlap.", + cols, + rows, + len(path_points), ) acquired_images: list[Image] = [] @@ -134,7 +147,9 @@ def scan_region( # 5. Execute Moves and Acquisitions for i, (tgt_x, tgt_y) in enumerate(path_points): - print(f"Moving to position {i + 1}/{len(path_points)} -> X: {tgt_x:.2f}, Y: {tgt_y:.2f}") + logger.debug( + "Moving to position %d/%d -> X: %.2f, Y: %.2f", i + 1, len(path_points), tgt_x, tgt_y + ) pausepoint() progress(int((i / total_len) * 100), f"Moving to point {i + 1}/{total_len}") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 4d7658c..b46d934 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -77,7 +77,14 @@ config-settings-package = { uc2-cam-py = { "cmake.define.MVS_ROOT" = "/opt/MVS" [tool.ruff.lint] -extend-select = ["ANN", "D1", "F401"] +# T20: no print in the backend, it logs (newswitch.logging_setup) +extend-select = ["ANN", "D1", "F401", "T20"] + +[tool.ruff.lint.per-file-ignores] +# Command-line tools: what they print is their result +"newswitch/cli.py" = ["T20"] +"newswitch/config_io/__main__.py" = ["T20"] +"scripts/*" = ["T20"] [tool.ruff] diff --git a/backend/tests/test_app_config.py b/backend/tests/test_app_config.py index 3fd3457..cc923cb 100644 --- a/backend/tests/test_app_config.py +++ b/backend/tests/test_app_config.py @@ -135,6 +135,28 @@ def test_camera_debug_window(write_base: BaseWriter) -> None: assert ImswitchConfig().camera_debug_window is True +def test_logging_settings(write_base: BaseWriter) -> None: + """Without the keys nothing goes to the terminal and everything to backend.log.""" + write_base() + settings = ImswitchConfig() + assert settings.log_to_terminal == "none" + assert settings.log_to_file == "debug" + assert settings.log_file_path == "backend.log" + + write_base(log_to_terminal="warn", log_to_file="none", log_file_path="logs/x.log") + settings = ImswitchConfig() + assert settings.log_to_terminal == "warn" + assert settings.log_to_file == "none" + assert settings.log_file_path == "logs/x.log" + + +def test_unknown_log_level_is_an_error(write_base: BaseWriter) -> None: + """Only fatal, error, warn, info, debug and none are accepted.""" + write_base(log_to_terminal="warning") + with pytest.raises(ValidationError, match="log_to_terminal"): + ImswitchConfig() + + def test_unknown_key_is_an_error(write_base: BaseWriter) -> None: """A misspelled key is reported instead of silently ignored.""" write_base(use_virutal_microscope=True) diff --git a/backend/tests/test_auth.py b/backend/tests/test_auth.py index 421b231..39a6aa6 100644 --- a/backend/tests/test_auth.py +++ b/backend/tests/test_auth.py @@ -7,8 +7,9 @@ """ import base64 +import time from pathlib import Path -from typing import Iterator +from typing import Callable, Iterator import pytest import yaml @@ -325,6 +326,31 @@ def test_stream_websocket_rejects_a_wrong_token(client: TestClient) -> None: assert excinfo.value.code == 1008 +def test_stream_websocket_accepts_an_in_band_token( + authed_app: FastAPI, client: TestClient, token: str +) -> None: + """An accepted auth frame subscribes the socket to a shared encoder. + + Zstd rather than H.264, so the test needs no ffmpeg. The encoder is checked rather + than received bytes: frames only flow while a live view runs. + """ + contexts = authed_app.state.agent.contexts + with client.websocket_connect("/stream/zstd/1") as websocket: + websocket.send_json({"type": "auth", "token": token}) + assert wait_until(lambda: "frame_broadcaster" in contexts) + assert wait_until(lambda: contexts["frame_broadcaster"].encoder_count == 1) + + +def wait_until(condition: Callable[[], bool], timeout: float = 5.0) -> bool: + """Poll `condition` until it holds; the server side runs on the client's portal.""" + deadline = time.monotonic() + timeout + while not condition(): + if time.monotonic() > deadline: + return False + time.sleep(0.05) + return True + + # ---------------------------------------------------------------- authenticator diff --git a/backend/tests/test_logging_setup.py b/backend/tests/test_logging_setup.py new file mode 100644 index 0000000..1517da3 --- /dev/null +++ b/backend/tests/test_logging_setup.py @@ -0,0 +1,191 @@ +"""Tests for `newswitch.logging_setup`: levels per output, the log file and its fallback.""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from newswitch import logging_setup +from newswitch.logging_setup import ( + UVICORN_LOGGERS, + configure_logging, + log_prints, + route_to_root, +) + + +@pytest.fixture(autouse=True) +def restore_logging(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: + """Run each test in a temporary folder and undo what `configure_logging` changed. + + Only the handlers `configure_logging` installed are removed: pytest attaches handlers of + its own to the root logger and removes them itself. + """ + monkeypatch.chdir(tmp_path) + root = logging.getLogger() + root_level = root.level + uvicorn = { + name: (list(logging.getLogger(name).handlers), logging.getLogger(name).level) + for name in UVICORN_LOGGERS + } + uvicorn_propagate = {name: logging.getLogger(name).propagate for name in UVICORN_LOGGERS} + yield + for handler in logging_setup._installed: + root.removeHandler(handler) + handler.close() + logging_setup._installed.clear() + root.setLevel(root_level) + logging.captureWarnings(False) + for name, (handlers, level) in uvicorn.items(): + library_logger = logging.getLogger(name) + library_logger.handlers[:] = handlers + library_logger.setLevel(level) + library_logger.propagate = uvicorn_propagate[name] + + +def _flush() -> None: + for handler in logging_setup._installed: + handler.flush() + + +def test_each_output_has_its_own_level(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + """``info`` keeps debug records off the terminal, while the file takes them.""" + log_file = tmp_path / "out.log" + configure_logging("info", "debug", str(log_file)) + log = logging.getLogger("test.levels") + log.debug("debug record") + log.info("info record") + _flush() + + terminal = capsys.readouterr().err + assert "info record" in terminal + assert "debug record" not in terminal + written = log_file.read_text(encoding="utf-8") + assert "debug record" in written + assert "INFO test.levels: info record" in written + + +def test_levels_filter_upwards(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + """``warn`` and ``fatal`` let only records of that level and above through.""" + log_file = tmp_path / "out.log" + configure_logging("warn", "fatal", str(log_file)) + log = logging.getLogger("test.upwards") + log.info("info record") + log.warning("warning record") + log.error("error record") + log.critical("critical record") + _flush() + + terminal = capsys.readouterr().err + assert "info record" not in terminal + assert "warning record" in terminal + assert "error record" in terminal + written = log_file.read_text(encoding="utf-8") + assert "error record" not in written + assert "critical record" in written + + +def test_none_writes_nothing(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + """``none`` for both: no terminal output, no file.""" + configure_logging("none", "none", "out.log") + logging.getLogger("test.none").error("an error") + _flush() + + assert capsys.readouterr().err == "" + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("file_path", [None, "", " "]) +def test_missing_path_falls_back(tmp_path: Path, file_path: str | None) -> None: + """Without a path the log goes to backend.log in the working directory, with a warning.""" + configure_logging("none", "debug", file_path) + logging.getLogger("test.fallback").info("after the fallback") + _flush() + + written = (tmp_path / "backend.log").read_text(encoding="utf-8") + assert "WARNING" in written + assert "no log_file_path given" in written + assert "after the fallback" in written + + +@pytest.mark.parametrize("file_path", ["missing/out.log", "."]) +def test_unusable_path_falls_back(tmp_path: Path, file_path: str) -> None: + """A path in a missing folder, or a folder, falls back; no folder is created.""" + configure_logging("none", "info", file_path) + _flush() + + written = (tmp_path / "backend.log").read_text(encoding="utf-8") + assert f"log_file_path '{file_path}' cannot be used" in written + assert not (tmp_path / "missing").exists() + + +def test_the_file_is_appended_and_rotated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A restart appends; a full file is rotated to backend.log.1.""" + log_file = tmp_path / "backend.log" + log_file.write_text("from the last run\n", encoding="utf-8") + configure_logging("none", "info", "backend.log") + assert log_file.read_text(encoding="utf-8").startswith("from the last run\n") + + monkeypatch.setattr(logging_setup, "MAX_LOG_FILE_BYTES", 500) + configure_logging("none", "info", "backend.log") + log = logging.getLogger("test.rotation") + for index in range(20): + log.info("line %d of the rotation test", index) + _flush() + + assert (tmp_path / "backend.log.1").is_file() + assert not (tmp_path / "backend.log.4").exists() + + +def test_a_second_call_replaces_the_handlers(tmp_path: Path) -> None: + """Calling it again neither duplicates records nor keeps the old file.""" + configure_logging("none", "info", "first.log") + configure_logging("none", "info", "second.log") + logging.getLogger("test.again").info("once") + _flush() + + assert (tmp_path / "second.log").read_text(encoding="utf-8").count("once") == 1 + assert "once" not in (tmp_path / "first.log").read_text(encoding="utf-8") + + +def test_uvicorn_loses_its_own_handlers() -> None: + """uvicorn's loggers propagate to the root logger, without handlers or levels of their own.""" + access = logging.getLogger("uvicorn.access") + access.addHandler(logging.StreamHandler()) + access.propagate = False + access.setLevel(logging.INFO) + + configure_logging("none", "none", None) + + for name in UVICORN_LOGGERS: + library_logger = logging.getLogger(name) + assert library_logger.handlers == [] + assert library_logger.propagate is True + assert library_logger.level == logging.NOTSET + + +def test_route_to_root() -> None: + """A library logger's own handlers are taken away and it propagates again.""" + library_logger = logging.getLogger("test.library") + library_logger.addHandler(logging.StreamHandler()) + library_logger.propagate = False + + route_to_root("test.library") + + assert library_logger.handlers == [] + assert library_logger.propagate is True + + +def test_log_prints(caplog: pytest.LogCaptureFixture, capsys: pytest.CaptureFixture) -> None: + """Every non-empty printed line becomes a record; nothing reaches stdout.""" + caplog.set_level(logging.DEBUG, logger="test.prints") + with log_prints(logging.getLogger("test.prints"), logging.DEBUG): + print({1: [], 2: []}) # noqa: T201 - what a library would print + print("two\nlines") # noqa: T201 + + assert [record.getMessage() for record in caplog.records] == ["{1: [], 2: []}", "two", "lines"] + assert all(record.levelno == logging.DEBUG for record in caplog.records) + assert capsys.readouterr().out == "" diff --git a/docs/encoder_bugreport.md b/docs/encoder_bugreport.md new file mode 100644 index 0000000..e94d624 --- /dev/null +++ b/docs/encoder_bugreport.md @@ -0,0 +1,268 @@ +# Bugreport: H.264-Livebild zwischen `FrameBroadcaster` und Frontend + +Stand: 2026-10-28, Branch `KC/CamAdapter` (Commit `6343890`). + +## Symptome + +Simulierte Kamera aus uc2_cam_py (`use_virtual_microscope: false`, +`static_config_path: camTest_config.yaml`), Frontend über `just dev`. Die Frames kommen in +`FrameBroadcaster._schedule_encoder_frames` an. Mit aktivem Livestream im Frontend: + +- es wird nur ein Ausschnitt des Bildes dargestellt, +- die Bildrate ist sehr niedrig, +- nach einiger Zeit friert das Bild ein. + +## Kurzfassung + +| Symptom | Ursache | Bug | +|---|---|---| +| Einfrieren | Der H.264-Bytestrom wird im Backend zerstückelt (ein Drittel der Bytes und praktisch alle Keyframes gehen verloren). Der Browser-Decoder bricht beim ersten beschädigten Frame ab, und das Frontend erholt sich davon nicht. | 1, 2 | +| Ausschnitt | Die Live-Ebene ist ≈ 328 Szeneneinheiten groß, die 2D-Kamera zeigt bei `zoom={5}` aber nur Canvas-Pixel / 5 Einheiten. | 3 | +| niedrige Bildrate | `koil.sleep(0.01)` dauert ≈ 100 ms: Die Live-Loop schafft 10–13 fps statt der 20 fps der Kamera. Dazu kommen beschädigte Frames und die feste Annahme von 30 fps im Player. | 4, 1, 5 | + +Das Backend selbst friert nicht ein: In 3 Minuten Messung lief der Strom mit konstant ≈ 13 fps +bis zum WebSocket. + +## Messungen + +Aufbau: Backend in-process mit camTest (Auth aus, `camera_debug_window: false`), Live-View per +`POST /start_live_view`, ein Python-WebSocket-Client an `/stream/h264/1`. An jeder Stufe wurde +mitgezählt, und der empfangene sowie der vollständige ffmpeg-Strom wurden mitgeschnitten. Die +Mitschnitte wurden anschließend in headless Chromium (Playwright) über JMuxer 2.1.0 abgespielt, +mit denselben Optionen wie in `useH264LiveTexture`. Die Skripte lagen im Session-Scratchpad und +sind nicht im Repository. + +| Stufe | Messwert (Lauf über 180 s) | +|---|---| +| Kamera (konfiguriert) | 20 fps, 640×480, `uint16` | +| `broadcast_sync` ohne Client | 10,0 fps (Median-Abstand 100 ms) | +| `broadcast_sync` mit Client | 13,3 fps (Median-Abstand 88 ms) | +| `H264Encoder.encode_frame` | Median 2,3 ms, max. 107 ms, läuft im Detektor-Thread | +| ffmpeg-Ausgabe | 10,0 MB in 3200 Chunks; 80 Keyframes, jeder mit SPS/PPS; 7 Slices pro Bild | +| in `EncoderSubscription` verworfen | 812 Chunks = 3,3 MB (33 %) | +| WebSocket empfangen | 2387 Chunks (genau 1 pro Frame), 6,7 MB | +| davon SPS/PPS | 1 von 80 (nur der allererste) | +| davon IDR-Slices | 81 von 560 | +| Bilder ohne ersten Slice | 733 von 2388 (31 %) | + +Abspielen im Browser (Mitschnitt über 25 s): + +- **Vollständiger ffmpeg-Strom:** 327 von 327 Frames decodiert, kein Fehler. +- **Beim Client angekommener Strom:** `video.error` Code 3, + `PIPELINE_ERROR_DECODE: Failed to send video packet for decoding` bei Medienzeit 0,627 s. + JMuxer meldet danach `InvalidStateError` und setzt sich zurück, ohne dass bis zum Ende des + Mitschnitts noch ein Frame decodiert wird. + +--- + +## 1 · H.264-Bytestrom wird in der Subscription-Queue zerstückelt (Hauptursache) + +**Beschreibung.** +[`EncoderSubscription`](../backend/newswitch/broadcasters/encoders/subscription.py#L29-L31) +hält genau einen Chunk (`asyncio.Queue(maxsize=1)`). Ist die Queue voll, verwirft +[`push_chunk`](../backend/newswitch/broadcasters/encoders/subscription.py#L56) den ältesten. +Für Zstd ist das vertretbar, weil dort ein Chunk ein ganzes Frame ist. Für H.264 sind es +4096-Byte-Stücke eines fortlaufenden Bytestroms +([`_read_output`](../backend/newswitch/broadcasters/encoders/h264_encoder.py#L133-L138)). + +Dass fast immer etwas verworfen wird, liegt an asyncio: `StreamReader.read(4096)` kehrt ohne +Rückgabe an den Event-Loop zurück, solange sein Puffer Daten enthält. Die Reader-Task schiebt +deshalb alle Chunks eines ffmpeg-Ausgabeblocks direkt hintereinander in die Queue, und der +WebSocket-Handler ([`liveview.py:185`](../backend/newswitch/routes/ws/liveview.py#L185)) kommt +erst danach wieder an die Reihe. Pro Frame überleben so nur die letzten ≤ 4096 Bytes, gemessen: +genau ein WebSocket-Chunk pro Frame. + +Folgen: + +- Keyframes (IDR mit SPS/PPS, meist deutlich größer als 4 KB) kommen praktisch nie an: 1 von 80. + Damit gibt es keinen Punkt, an dem sich der Decoder erholen kann. +- `-tune zerolatency` lässt x264 jedes Bild in 7 Slices aufteilen (sliced threads). Fehlt der + erste Slice eines Bildes (31 % der Bilder), hängt JMuxer die restlichen Slices an das + vorherige Sample. Es entstehen ungültige Samples, und der Decoder bricht ab (siehe Bug 2). + Ein fehlertoleranterer Decoder würde stattdessen nur Teile des Bildes (einzelne Streifen) + aktualisieren, was ebenfalls wie ein Ausschnitt aussehen kann. + +Bereits als Risiko in OP #30 und +[`LIVE-IMAGE-PIPELINE.md`](../backend/docs/LIVE-IMAGE-PIPELINE.md) §8 erwähnt. Hiermit ist +bestätigt, dass dies die Hauptursache ist. + +**Fixvorschlag.** + +1. Sofortmaßnahme: Der H.264-Pfad darf keine Bytes verwerfen. Eine Queue ohne Größenlimit + (oder ein großes Limit) für H.264-Subscriptions, z. B. über einen Parameter `maxsize` von + `EncoderSubscription`, den der H.264-Encoder anders setzt als der Zstd-Encoder. Zusätzlich + in größeren Blöcken lesen (`read(65536)`), das spart WebSocket-Nachrichten. +2. Robuste Lösung: ganze Frames (Access Units) statt 4096-Byte-Stücken verteilen. Dazu z. B. + ffmpeg mit `-x264-params aud=1` starten und am Access Unit Delimiter trennen. Läuft die Queue + eines langsamen Clients über, alles bis zum nächsten IDR (mit SPS/PPS) verwerfen. Der Client + verliert dann ganze Frames, aber nie Teile davon. Ein neuer Client bekommt erst ab dem + nächsten Keyframe Daten. Alternativ SPS/PPS und den letzten IDR zwischenspeichern und beim + Subscribe zuerst senden. +3. Test: einen Ausgabeblock > 4096 Bytes einspeisen und prüfen, dass alle Bytes vollständig und + in Reihenfolge beim Subscriber ankommen. + +Die Zstd-Route verwendet `use_delta=True` +([`liveview.py:125`](../backend/newswitch/routes/ws/liveview.py#L125)). Dort zerstört ein +verworfenes Delta-Frame die Rekonstruktion ebenfalls. Das betrifft das Frontend derzeit nicht, +weil der Zstd-Pfad dort nicht genutzt wird. + +## 2 · Frontend erholt sich nicht von einem Decoder-Fehler (dauerhaftes Einfrieren) + +**Beschreibung.** +[`useH264LiveTexture`](../frontend/src/components/stage/hooks/useH264LiveTexture.tsx#L66-L98) +setzt weder JMuxers `onError` noch einen Listener für `error` am `