Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions backend/backend_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -678,11 +678,17 @@ def start_backend(host: str, port: int) -> tuple[uvicorn.Server, threading.Threa
SystemExit: The server stopped before it was up (e.g. the port is taken).
"""
from newswitch.app import create_app, load_app_config
from newswitch.logging_setup import configure_logging

app = create_app(load_app_config())
config = load_app_config()
configure_logging(config.log_to_terminal, config.log_to_file, config.log_file_path)
app = create_app(config)
# No access log: every call of the console would add a line (with --verbose: to its menu).
# log_config=None: uvicorn would otherwise put its own handlers back.
server = uvicorn.Server(
uvicorn.Config(app, host=host, port=port, log_level="warning", access_log=False)
uvicorn.Config(
app, host=host, port=port, log_level="warning", access_log=False, log_config=None
)
)
thread = threading.Thread(target=server.run, name="backend", daemon=True)
thread.start()
Expand Down
15 changes: 12 additions & 3 deletions backend/base_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Host and port are not set here. They come from BACKEND_HOST / BACKEND_PORT in the
# repository's root .env (deployment via docker is still an open point).

use_virtual_microscope: true
use_virtual_microscope: false
# SQLite file of the rekuest agent (state history); relative to the working directory
db_path: agent_data.db
available_cubes: [cube1, cube2, cube3]
Expand All @@ -15,8 +15,8 @@ schema_dir: configs/schemas

# Device configuration; bare names are resolved against config_dir.
# newswitch never writes static_config_path; changes always go into persistent_config_path.
static_config_path: newswitch-config.yaml
persistent_config_path: newswitch-config.persistent.yaml
static_config_path: camTest_config.yaml
persistent_config_path: camTest_config.persistent.yaml
# true: read static_config_path; on shutdown persistent_config_path is overwritten
# with it plus the runtime changes
# false: read (and write back) persistent_config_path; on first start it is created
Expand All @@ -33,3 +33,12 @@ camera_recording_dir: data/recordings
# the window always has one; this writes nothing to disk. Without an X11 display only an
# error is logged. Set false on headless machines.
camera_debug_window: true

# Logging of the backend (the frontend logs in the browser). Each output gets records from
# its level up: fatal | error | warn | info | debug, or none for no output at all.
log_to_terminal: none
log_to_file: debug
# Log file, relative to the backend's working directory (backend/ with `just dev`). It is
# appended to and rotated at 10 MB (backend.log.1 ... .3 are kept). Missing, or not openable
# (e.g. its folder does not exist): backend.log is used, with a warning in it.
log_file_path: backend.log
14 changes: 14 additions & 0 deletions backend/docs/DEVELOPER_OVERVIEW.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,20 @@ A device kind missing from the file, or `config_file=None` (the tests), keeps th
built-in virtual defaults. On shutdown exposure, gain, wheel positions and stage position
are written back; comments stay intact. Details: `backend/configs/README.md`.

### Logging (`newswitch/logging_setup.py`)

The backend does not print; every module logs through `logging.getLogger(__name__)` (ruff
rule T20; only the command-line tools `newswitch/cli.py`, `newswitch/config_io/__main__.py`
and `scripts/` print their results). `base_config.yaml` sets one level for the terminal
(`log_to_terminal`) and one for the log file (`log_to_file`): `fatal`, `error`, `warn`,
`info`, `debug` or `none`. The file (`log_file_path`, relative to the working directory) is
appended to and rotated at 10 MB; if it cannot be opened, `backend.log` is used with a
warning. `configure_logging` runs in the entry points (`main.py`, `newswitch.app.main`,
`backend_cli.py`), not in `create_app`, so tests write no log files. It also takes away
uvicorn's own handlers; `route_to_root` and `log_prints` do the same for libraries that
attach handlers or print (uc2rest in `managers/uc2/rest_bus.py`). Not covered: uvicorn's
reloader process under `--reload`, prints of other libraries, native output.

### Registered functions

Everything the frontend can call lives in `newswitch/registers/`: one module per device
Expand Down
28 changes: 12 additions & 16 deletions backend/docs/LIVE-IMAGE-PIPELINE.md
Original file line number Diff line number Diff line change
Expand Up @@ -277,8 +277,8 @@ sequenceDiagram
H->>H: accept
BR->>H: auth frame with token
H->>H: authenticate_stream_websocket, close 1008 if invalid
H->>H: get_broadcaster_from_websocket
Note over H: Fails today, the agent has no get_context_for_type (issue 1)
H->>H: await_broadcaster, context frame_broadcaster from agent.contexts
Note over H: waits up to 10 s for the agent's startup hooks, then closes 1011
H->>FB: get_subscription_(slot, H264EncoderConfig())
FB->>EN: create if no encoder for (slot, config), store in _encoders
FB->>EN: astart() if not running
Expand All @@ -303,7 +303,7 @@ sequenceDiagram
end

BR--xH: disconnect (LivePlane unmounts, e.g. live view stopped)
Note over H: noticed only when the next send_bytes fails (issue 3)
Note over H: noticed only when the next send_bytes fails (issue 1)
H->>FB: arelease_subscription(subscription)
FB->>EN: aunsubscribe, astop() if it was the last subscriber
EN->>FF: kill
Expand Down Expand Up @@ -396,17 +396,12 @@ after `onopen` the hook sends the auth frame (`authFrame()`, the token). A close

Tracked in `.claude/OpenPoints.md`.

1. **The stream route crashes on every connection (OP #28).**
`get_broadcaster_from_websocket` (`liveview.py:75`) calls `agent.get_context_for_type(...)`,
which rekuest_next 2.3.1 does not have (it offers `agent.contexts[key]` /
`aget_context(key)`; the key is `frame_broadcaster`). The handler raises `AttributeError`
before `get_subscription_`. No encoder is ever created, and no image reaches the frontend.
Reproduced with camTest: 0 encoders and 0 bytes with a connected, authenticated client.
`tests/test_auth.py` only tests the rejected path.
2. **ffmpeg's stderr is never read (OP #29).** `stderr=PIPE` (`h264_encoder.py:118`) without a
reader: ffmpeg writes progress lines continuously, and once the pipe buffer (about 64 KB) is
full, ffmpeg blocks and the stream freezes.
3. **Further risks (OP #30):**
Fixed on 2026-10-27: the stream route crashed on every connection (it called
`agent.get_context_for_type`, which rekuest_next 2.3.1 does not have), and ffmpeg's unread
stderr pipe froze the H.264 stream after a few minutes (ffmpeg now runs with
`-loglevel error -nostats`, stderr discarded).

1. **Further risks (OP #30):**
- Thread safety. `encode_frame` runs on the worker thread: H.264 calls
`StreamWriter.write` there, Zstd calls `asyncio.Queue.put_nowait`. Neither is thread-safe.
- Dropped H.264 chunks. The queue keeps one 4096-byte chunk and drops the oldest. Dropping
Expand All @@ -423,7 +418,7 @@ Tracked in `.claude/OpenPoints.md`.
`uint8` full frames (unused today).
- Dead frontend code. `components/microscope/LiveView.tsx` is not mounted anywhere; its
`StreamingView` connects to `VITE_WEBSOCKET_URL + "/video"`, a route that does not exist.
4. **Frame rate limited by `koil.sleep` (OP #3).** The live loop's 10 ms pause takes about
2. **Frame rate limited by `koil.sleep` (OP #3).** The live loop's 10 ms pause takes about
140 ms, which limits the stream to about 7 fps.

---
Expand All @@ -442,4 +437,5 @@ Tracked in `.claude/OpenPoints.md`.
`--verbose` is given.
- **Browser side:** DevTools → Network → WS: is there a connection to `/stream/h264/<slot>`,
on the right port, and with which close code? 1006 (no close frame) points to a
server-side exception, 1008 to a rejected token.
server-side exception, 1008 to a rejected token, 1011 to a FrameBroadcaster the agent did
not provide within 10 s (startup hooks failed or still running).
11 changes: 10 additions & 1 deletion backend/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,29 @@

Run directly (``python main.py``, e.g. in the debugger) the app serves on BACKEND_HOST /
BACKEND_PORT from the environment, like `newswitch.app.main`.

Logging follows log_to_terminal / log_to_file / log_file_path of the settings. Under
``uvicorn main:app`` uvicorn sets up its own logging first and imports this module afterwards,
so the settings win.
"""

import os

from newswitch.app import create_app, load_app_config
from newswitch.logging_setup import configure_logging

app = create_app(load_app_config())
config = load_app_config()
configure_logging(config.log_to_terminal, config.log_to_file, config.log_file_path)
app = create_app(config)


if __name__ == "__main__":
import uvicorn

# log_config=None: uvicorn would otherwise put its own handlers back
uvicorn.run(
app,
host=os.environ.get("BACKEND_HOST", "0.0.0.0"),
port=int(os.environ.get("BACKEND_PORT", "8099")),
log_config=None,
)
33 changes: 22 additions & 11 deletions backend/newswitch/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
from newswitch.protocols.objective import ObjectiveLens
from newswitch.protocols.uc2 import UC2BusManager, UC2State
from newswitch import registers # noqa: F401 (registration side effects: every action)
from newswitch.logging_setup import LogLevel, configure_logging
from newswitch.managers.expanse_manager import ExpanseManager
from newswitch.managers.metadata_manager import MetadataManager
from newswitch.protocols import (
Expand Down Expand Up @@ -163,6 +164,13 @@ class ImswitchConfig(BaseSettings):
"""Show every camera's newest image in a window of uc2_cam_py (debugging without the
frontend); the cameras then acquire from startup on."""

log_to_terminal: LogLevel = "none"
"""Lowest level of the log records written to the terminal (see `logging_setup`)."""
log_to_file: LogLevel = "debug"
"""Lowest level of the log records written to `log_file_path`."""
log_file_path: Optional[str] = "backend.log"
"""Log file, relative to the working directory. Missing or not openable: backend.log."""

# UC2 hardware transport (used when use_virtual_microscope is False).
# "canopen" (preferred, multi-node CAN bus) or "rest" (single ESP32 master
# over serial JSON). Both are plug-in replacements behind UC2BusManager.
Expand Down Expand Up @@ -361,21 +369,21 @@ async def provide_managers( # TODO: can we make this adaptive so that we read t
app_context = ImswitchConfig()

mode = "virtual" if app_context.use_virtual_microscope else "real hardware"
print(f"Initializing {mode} microscope managers")
print(f"UC2 transport: {app_context.uc2_transport}")
logger.info("Initializing %s microscope managers", mode)
logger.info("UC2 transport: %s", app_context.uc2_transport)
if app_context.uc2_transport == "canopen":
print(
"CAN interface: "
f"{app_context.uc2_can_interface}, "
f"channel={app_context.uc2_can_channel}, "
f"port={app_context.uc2_can_port}"
logger.info(
"CAN interface: %s, channel=%s, port=%s",
app_context.uc2_can_interface,
app_context.uc2_can_channel,
app_context.uc2_can_port,
)

frame_broadcaster = FrameBroadcaster()

config = app_context
if config.use_virtual_microscope:
print("Using virtual microscope managers.")
logger.info("Using virtual microscope managers.")

illumination_state = IlluminationState()

Expand Down Expand Up @@ -529,7 +537,7 @@ async def provide_managers( # TODO: can we make this adaptive so that we read t
hook_manager.register_hook(protocols.SoftwareAutofocusHook, software_autofocus_hook)
hook_manager.register_hook(
protocols.ZCalibrationHook,
lambda hook, context: print("Z-calibration hook executed"),
lambda hook, context: logger.info("Z-calibration hook executed"),
)

expanse_state = protocols.ExpanseState()
Expand Down Expand Up @@ -626,9 +634,9 @@ def save_device_config(
adapters.sync_runtime_state(
cfg_file.config, camera_state, objective_state, filter_bank_state, stage_state
)
print(f"Saved device configuration to {cfg_file.save()}")
logger.info("Saved device configuration to %s", cfg_file.save())
except config_io.ConfigError as exc:
print(f"Device configuration not saved: {exc}")
logger.warning("Device configuration not saved: %s", exc)


@shutdown
Expand Down Expand Up @@ -758,11 +766,14 @@ def main() -> None:
settings come from `load_app_config`, so ``NEWSWITCH_CONFIG`` applies as for main.py.
"""
config = load_app_config()
configure_logging(config.log_to_terminal, config.log_to_file, config.log_file_path)
app = create_app(config)
# log_config=None: uvicorn would otherwise put its own handlers back
uvicorn.run(
app,
host=os.environ.get("BACKEND_HOST", "0.0.0.0"),
port=int(os.environ.get("BACKEND_PORT", "8099")),
log_config=None,
)


Expand Down
8 changes: 7 additions & 1 deletion backend/newswitch/broadcasters/encoders/h264_encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,14 @@ async def astart(self) -> None:
if self._running:
return

# Nobody reads ffmpeg's stderr, and a full pipe would block ffmpeg and freeze the
# stream: keep it quiet (no banner, no progress line) and discard what is left.
command = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-nostats",
"-y",
"-f",
"rawvideo",
Expand Down Expand Up @@ -115,7 +121,7 @@ async def astart(self) -> None:
*command,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
self._running = True
self._reader_task = asyncio.create_task(self._read_output())
Expand Down
14 changes: 14 additions & 0 deletions backend/newswitch/broadcasters/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
- Shared encoders for clients with matching configurations
"""

import logging
from typing import Optional
import asyncio
import threading
Expand All @@ -23,6 +24,8 @@
from newswitch.broadcasters.encoders.config import EncoderConfig
from newswitch.broadcasters.encoders.base import EncoderSubscriptionProtocol

logger = logging.getLogger(__name__)


@context
class FrameBroadcaster:
Expand Down Expand Up @@ -136,6 +139,17 @@ async def arelease_subscription(self, subscription: EncoderSubscriptionProtocol)
def _schedule_encoder_frames(self, detector_slot: int, frame: np.ndarray) -> None:
"""Schedule frame encoding for active encoders of one detector slot."""

if logger.isEnabledFor(logging.DEBUG): # max/min of every frame are not free
logger.debug(
"Scheduling frame for detector slot %s with shape %s, dtype %s, max %s, "
"min %s, encoders: %d",
detector_slot,
frame.shape,
frame.dtype,
frame.max(),
frame.min(),
len(self._encoders),
)
for (slot, _), encoder in self._encoders.items():
if slot != detector_slot:
continue
Expand Down
5 changes: 4 additions & 1 deletion backend/newswitch/hooks/software_autofocus.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"""Software autofocus hook: refocus using the shared autofocus sweep core."""

import logging
from newswitch import protocols
from newswitch.routines.autofocus import autofocus_sweep

logger = logging.getLogger(__name__)

DEFAULT_Z_RANGE = 20.0 # micrometers, centered on the current Z
DEFAULT_STEPS = 11

Expand All @@ -17,4 +20,4 @@ def software_autofocus_hook(hook: protocols.Hook, context: protocols.HookContext
]
best_z, metrics = autofocus_sweep(stage, context.detector_manager, z_positions)
stage.move(z=best_z, is_absolute=True)
print(f"Autofocus hook: best z={best_z:.2f} (metric {max(metrics):.1f})")
logger.info("Autofocus hook: best z=%.2f (metric %.1f)", best_z, max(metrics))
Loading
Loading