diff --git a/.gitignore b/.gitignore index 6502dc232..9b0440ae2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.py[cod] venv +.venv/ .ipynb_checkpoints/ tools/ # mypy @@ -45,6 +46,7 @@ pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports +out/ htmlcov/ .tox/ .coverage diff --git a/imswitch/imcontrol/_test/hil/README.md b/imswitch/imcontrol/_test/hil/README.md new file mode 100644 index 000000000..9b722e8cf --- /dev/null +++ b/imswitch/imcontrol/_test/hil/README.md @@ -0,0 +1,167 @@ +# Hardware-in-the-loop (HIL) tests + +Tests of a running ImSwitch on a real microscope, through its HTTP API only. +They sit next to `../unit/` (no server) and `../api/` (headless server), but never import ImSwitch: they need only `pytest`, `requests`, `pillow`, `numpy`, `reportlab` and `pypdf`. + +**They move the stage and switch light.** +Each test file says at the top of its docstring what it checks and what it does to the rig. + +| Test | Checks | Hardware | +|---|---|---| +| `board_link` | ImSwitch talks to the UC2 board, and the board runs the master firmware | reads | +| `firmware_server` | the firmware server has firmware files, and each board runs the server's version | reads | +| `camera_capture` | each camera takes a picture as large as its sensor | reads | +| `motor_axis_move` | each axis moves 10 µm and back, and ImSwitch reports the position it moved to | **moves** (10 µm) | +| `stage_homing` | "Safe Global Homing" (`startFrameHoming`) homes Z, X and Y and reports each as done | **moves** (to the end stops, then to the transport position) | +| `lightsource` | each laser and LED switches on and off; every camera sees each LED go on and off | **light** | +| `ledmatrix` | every camera sees the LED matrix go on and off | **light** | +| `frame_calibration` | the FRAME calibration runs from start to end; needs the calibration target in the holder | **moves** (homes, turret), **light**, **writes the setup** (backup: `setup_before_frame_calibration.json`) | + +Hardware a setup does not have skips its tests. +Hardware that is there but does not work fails. +A camera of the setup that ImSwitch serves as a mock fails `camera_capture`: the real one did not start. + +They run in the order of the table: the boards, the cameras, the motors, the lights, then the calibration. +The homing ends at the transport position, and the light tests run there. +The order is the list `ORDER` in `conftest.py`. + +A light test has one case per light, not one per camera. +It passes only when every camera of the setup sees the light go on and go off again. +A camera that ImSwitch serves as a mock, that cannot report its status or that cannot give frames is left out and named in the report. + +Some tests do not start after a failure; they are skipped and say which test failed. +The table is `NEEDS` in `conftest.py`: + +- `stage_homing` needs `motor_axis_move`. +- `frame_calibration` needs `board_link`, `camera_capture`, the two motor tests and the two light tests. + It does not need `firmware_server`: a board on another firmware version fails that test, but can still be calibrated. + Only `--sync-firmware` stops every test, when the flashing fails. + +`motor_axis_move` compares positions, which are the firmware's step count. +`stage_homing` presses the same button as the FRAME settings page and checks the same per-axis "done". +That "done" says that the homing command of the axis returned, not that its end stop was pressed. +The turret (A) is homed by the calibration. +The calibration homes again and checks every axis, reads the position back after every move, follows X and Y in the observation camera, focuses with Z and switches the objective. +A step the calibration skips (e.g. the second objective on a rig with one) is listed in the test's output as not tested. + +## Run + +```bash +python -m imswitch.imcontrol._test.hil REPORT_DIR [TEST ...] [--sync-firmware] +``` + +```bash +python -m imswitch.imcontrol._test.hil out # everything +python -m imswitch.imcontrol._test.hil out board_link camera_capture # only these tests +python -m imswitch.imcontrol._test.hil out --sync-firmware # FLASHES every board first +``` + +`TEST` is the name from the table: `board_link` runs `test_board_link.py`. + +It waits until ImSwitch answers and its setup is readable. +A camera that does not start is a problem in the report; the tests run anyway. +`REPORT_DIR` receives: + +| File | Holds | +|---|---| +| `hil-report.pdf` | the report to read: what was tested, one line per test, then every test with what failed, its notes, its cases and its photos, then the PDFs the tests appended (the calibration's own report) | +| `summary.md` | the setup, the counts, a table with one row per test file, one line per failure and the notes of the tests, for the CI run's page | +| `setup_before_frame_calibration.json` | the setup as it was before the calibration; only when the calibration ran | +| `frame_calibration.pdf` | the calibration's own report, also appended to `hil-report.pdf`; only when the calibration ran | + +Both reports name the setup file the rig ran with. +A test finds the folder in the environment variable `HIL_REPORT_DIR`. +Use a new folder for every run: the two reports are replaced, but a file a test saved there in an older run stays. + +While the tests run, the log shows one line per test with the number of tests done (`[12/31]`). +The calibration takes minutes and prints a line when each of its steps starts and ends (`calibration 4/10 First objective: ...`). + +Exit code: `0` all passed, `1` a test failed, `2` the tests could not run (not ready, firmware sync failed, unknown test, everything skipped, the PDF could not be written). + +The test rig (os-rpi, `.github/hil/`) runs exactly this command inside the ImSwitch container. +That one line is all the two repositories share, so the tests always belong to the ImSwitch version under test. + +From a laptop against a rig, without ImSwitch installed: + +```bash +IMSWITCH_URL=http:///imswitch python imswitch/imcontrol/_test/hil out board_link +``` + +While writing a test, plain pytest works too and shows more: + +```bash +cd imswitch/imcontrol/_test/hil +IMSWITCH_URL=http:///imswitch pytest test_camera_capture.py -s +``` + +## Files + +Every `test_*.py` is one test and stands on its own: its settings are at its top, its helper functions below them. +No test imports another one. +Deleting a test is deleting its file. + +What the tests share has nothing to do with what they check: + +| File | Holds | +|---|---| +| `hil_client.py` | HTTP (`get_json`, `post_json`, `wait_until`), `setup()`, `parametrize_from_rig` | +| `hil_report.py` | `attach(label, image)`, `note(text)`, `append_pdf(title, data)`, the results of the run, `summary.md` | +| `hil_pdf.py` | `hil-report.pdf` | +| `conftest.py` | the order of the tests (`ORDER`); which test needs which (`NEEDS`); `live`; marks the tests `hardware`; skips everything when ImSwitch is not reachable | +| `__main__.py` | the command above | +| `firmware_sync.py` | `--sync-firmware` | + +## Which hardware a test uses + +There is no list of the rig's hardware here. +The tests read it from the setup file ImSwitch runs with: `ImSwitchConfig/imcontrol_setups/.json`, where `` is the `setupFileName` in `ImSwitchConfig/config/imcontrol_options.json`. +`client.setup_file()` returns the name, `client.setup()` the content as a dict. +ImSwitch parses the file before it answers, so a field the file leaves out comes back with ImSwitch's default (e.g. `"forAcquisition": false`). + +| Setup | Used for | +|---|---| +| `detectors`, `forAcquisition` | the cameras; `true` is an acquisition detector, `false` the observation camera | +| `lasers`, `valueRangeMax` | the lasers and LEDs, and the value they are switched on with | +| `LEDMatrixs` | whether there is an LED matrix | +| `positioners`, `axes` | the stage axes | +| `objective` | whether there is an objective turret | + +So a rig with other hardware needs another setup file in ImSwitch, not other tests. + +## Writing a test + +Copy the test that is closest to the new one, rename it `test_.py`, and change it. +It runs with the others from then on, after the tests listed in `ORDER` (`conftest.py`); add its name there to give it another place. +A test file has: + +1. a docstring: first a paragraph saying in plain words what it checks, then what it does to the rig (MOVES REAL HARDWARE, SWITCHES LIGHT, Only reads) and anything else worth knowing; +2. its settings, as constants; +3. its helper functions and fixtures; +4. the tests, whose steps read top to bottom. + +- Call ImSwitch with `client.get_json("/Controller/method", {"param": value})`. +- Take what the rig has from `client.setup()`, and thresholds, distances and timeouts from the constants at the top of the file. +- Add to the test's part of the report with `attach("label", image)` for a photo and `hil_report.note("...")` for a line of text. + `hil_report.append_pdf("title", data)` puts a whole PDF behind the pages of the tests. + The report code knows no test by name. +- The first paragraph of the docstring is what the report shows under "What it checks" (`hil_report.py`). + Write it as you would say it to a colleague. +- A test that runs for minutes takes the fixture `live` and prints its progress with `live("step 4/10 ...")`; a plain `print` only shows when the test fails. +- Whatever a test switches on or moves, it puts back, also when it fails (`try/finally`, or a fixture with `yield`). +- Code that a second test needs is copied, not shared: a test must stay readable and removable by itself. + +## The calibration in the report + +The test waits until the calibration's thread has ended, then fetches its PDF with `getFrameCalibrationReportPdf` and checks the run's `startedAt` before and after, so the report of another run is never taken. +The PDF is appended to `hil-report.pdf` as ImSwitch wrote it (`hil_report.append_pdf`). +It is also saved on its own as `frame_calibration.pdf` in the report folder, so it is there even when `hil-report.pdf` could not be written. +When it cannot be fetched, the `frame_calibration` part of the report shows that as an error. + +The fixtures of `test_frame_calibration.py` put the rig back, each on its own: the calibration is stopped, the light goes off (the calibration leaves it on), the settings go back, and then the report is collected. + +The calibration's result stays in the instrument's setup, also after os-rpi has restored the deployment. +A calibration that fails may have written a part of it; `setup_before_frame_calibration.json` is there to go back by hand. +Nothing is rolled back automatically. + +os-rpi passes three environment variables that the report shows: `HIL_TARGET` (the image or pallet under test), `HIL_IMAGES` (every image of the pallet under test, one per line) and `HIL_RUN_URL` (the Actions run). +A run from a laptop has none of them. diff --git a/imswitch/imcontrol/_test/hil/__main__.py b/imswitch/imcontrol/_test/hil/__main__.py new file mode 100644 index 000000000..da51144b0 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/__main__.py @@ -0,0 +1,140 @@ +"""Run the hardware-in-the-loop (HIL) tests against a live ImSwitch and write a report. + + python -m imswitch.imcontrol._test.hil REPORT_DIR [TEST ...] [--sync-firmware] + + python -m imswitch.imcontrol._test.hil out # everything + python -m imswitch.imcontrol._test.hil out board_link camera_capture # only these tests + python -m imswitch.imcontrol._test.hil out --sync-firmware # FLASHES every board first + +TEST is the name of a test file without "test_" and ".py": board_link is test_board_link.py. +Without TEST every test runs, which MOVES THE STAGE AND SWITCHES LIGHT. +REPORT_DIR receives hil-report.pdf and summary.md; +both name the ImSwitch setup file (ImSwitchConfig/imcontrol_setups/.json) the rig ran with. +Use a new folder for every run: only these two files are replaced, +a file a test saved there in an older run stays. + +Exit code: 0 all passed, 1 a test failed, 2 the tests could not run. +The test rig (os-rpi) runs exactly this command inside the ImSwitch container. +""" +import argparse +import os +import sys +from pathlib import Path + +HERE = Path(__file__).parent +sys.path.insert(0, str(HERE)) # so the modules of this folder import as they do under pytest + +import pytest # noqa: E402 +import requests # noqa: E402 + +import firmware_sync # noqa: E402 +import hil_client as client # noqa: E402 +import hil_report # noqa: E402 + +PASSED, FAILED, COULD_NOT_RUN = 0, 1, 2 + +READY_API_TIMEOUT = 180 # seconds ImSwitch may take to answer +READY_DETECTOR_TIMEOUT = 300 # seconds each camera may take to give its first frame + + +def can_snap(detector, for_acquisition): + """Whether the camera gives a frame yet; right after a start its driver may still load.""" + try: + if for_acquisition: + return client.get("/RecordingController/snapNumpyToFastAPI", params={ + "detectorName": detector, "resizeFactor": 0.1}).status_code == 200 + # 400: no observation camera is connected. + # Waiting does not change that, and the tests that need it skip. + return client.post("/ExperimentController/snapOverviewImage", timeout=30, params={ + "slot_id": "1", "camera_name": "hil_ready_check"}).status_code in (200, 400) + except requests.RequestException: + return False + + +def wait_for_imswitch(): + """Wait until ImSwitch answers and its setup can be read; the tests are collected from it.""" + assert client.wait_until(client.reachable, READY_API_TIMEOUT, interval=5), ( + f"ImSwitch did not answer at {client.IMSWITCH_URL} within {READY_API_TIMEOUT}s") + client.setup() + + +def prepare_cameras(): + """Give the cameras time to start. Returns {camera: what went wrong}, empty when all run.""" + problems = {} + for name, detector in client.setup()["detectors"].items(): + try: + acquisition = detector["forAcquisition"] + if acquisition: + result = client.post_json("/LiveViewController/startLiveView", + {"detectorName": name}, timeout=60) + assert result.get("status") in ("success", "already_running"), result + assert client.wait_until(lambda: can_snap(name, acquisition), + READY_DETECTOR_TIMEOUT, interval=5), "no frame before timeout" + except (requests.RequestException, AssertionError) as error: + problems[name] = str(error) + return problems + + +def give_up(report_dir, problem): + """Write a report that only says why the tests did not run.""" + print(f"[hil] not running the tests: {problem}") + hil_report.record_problem("startup", problem) + hil_report.write_reports(report_dir, "unavailable") + return COULD_NOT_RUN + + +def main(): + parser = argparse.ArgumentParser(prog="python -m imswitch.imcontrol._test.hil") + parser.add_argument("report_dir", help="folder for hil-report.pdf and summary.md") + parser.add_argument("tests", nargs="*", + help="tests, e.g. board_link camera_capture; default all") + parser.add_argument("--sync-firmware", action="store_true", + help="FLASH every board to the firmware server's version first") + args = parser.parse_args() + + report_dir = Path(args.report_dir) + report_dir.mkdir(parents=True, exist_ok=True) + os.environ["HIL_REPORT_DIR"] = str(report_dir.resolve()) + hil_report.reset() + # A run that dies must not leave the report of an older run behind. + for name in ("hil-report.pdf", "summary.md"): + (report_dir / name).unlink(missing_ok=True) + files = [HERE / f"test_{name}.py" for name in args.tests] + unknown = [name for name, file in zip(args.tests, files) if not file.is_file()] + if unknown: + known = " ".join(sorted(file.stem[len("test_"):] for file in HERE.glob("test_*.py"))) + return give_up(report_dir, f"no such test: {' '.join(unknown)}. There are: {known}") + + try: + wait_for_imswitch() + print(f"[hil] setup: {client.setup_file()}", flush=True) + if args.sync_firmware: + firmware_sync.sync() + except (requests.RequestException, AssertionError, firmware_sync.SyncFailed) as error: + return give_up(report_dir, str(error)) + + # A camera that does not start is a problem in the report; the tests run anyway. + for name, error in prepare_cameras().items(): + hil_report.record_problem(f"camera preparation: {name}", error) + + code = pytest.main( + [*[str(file) for file in files or [HERE]], + "-c", str(HERE / "pytest.ini"), "-ra"], + plugins=[hil_report.ReportPlugin(report_dir, client.setup_file())]) + + counts = hil_report.counts() + print("[hil] " + " · ".join(f"{n} {status}" for status, n in counts.items())) + if hil_report.report_error: + return COULD_NOT_RUN + if counts["failed"] or counts["error"]: + return FAILED + if code != pytest.ExitCode.OK: + return COULD_NOT_RUN # e.g. a test file that does not import + if not counts["passed"]: + print("[hil] every test skipped: nothing on this rig was actually tested") + return COULD_NOT_RUN + return PASSED + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/imswitch/imcontrol/_test/hil/conftest.py b/imswitch/imcontrol/_test/hil/conftest.py new file mode 100644 index 000000000..579fc20c7 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/conftest.py @@ -0,0 +1,86 @@ +"""What applies to every HIL test: the order they run in, and when they do not run at all. + +Nothing here knows what a test checks. +""" +from pathlib import Path + +import pytest + +import hil_client as client + +# The order the test files run in. A test file that is not listed runs after these. +ORDER = [ + "board_link", "firmware_server", # 1. the boards and their firmware versions + "camera_capture", # 2. the cameras + "motor_axis_move", # 3. the motors: 10 µm steps, + "stage_homing", # homing, ending at the transport position, + "lightsource", "ledmatrix", # 4. the lights, at the transport position + "frame_calibration", # 5. the calibration +] +# A test does not start when one of these failed before it. +# firmware_server is in no list: a board on another firmware version fails that test, +# but can still move and be calibrated. +# Only --sync-firmware stops every test, when flashing fails (__main__.py). +NEEDS = { + "stage_homing": ["motor_axis_move"], + "frame_calibration": ["board_link", "camera_capture", "motor_axis_move", "stage_homing", + "lightsource", "ledmatrix"], +} + +_failed = [] # the names of the tests that failed so far in this run + + +def name_of(item_or_report): + """ "board_link" for a test of test_board_link.py.""" + file = item_or_report.nodeid.split("::")[0] # "test_board_link.py::test_uc2_board_connected" + return Path(file).stem[len("test_"):] + + +def pytest_collection_modifyitems(items): + """Put the tests in ORDER, and mark every HIL test ``hardware``. + + The mark lets ``pytest -m "not hardware"`` leave the suite out. + """ + # The sort keeps the order of the tests inside a file. + items.sort(key=lambda item: ORDER.index(name_of(item)) if name_of(item) in ORDER + else len(ORDER)) + for item in items: + if Path(__file__).parent in Path(item.fspath).parents: + item.add_marker(pytest.mark.hardware) + + +def pytest_runtest_logreport(report): + """Called with every test's result: remember the tests that failed.""" + if report.failed: + _failed.append(name_of(report)) + + +def pytest_runtest_setup(item): + """Called before a test and its fixtures start: skip it when a test it needs failed.""" + failed = [name for name in NEEDS.get(name_of(item), []) if name in _failed] + if failed: + pytest.skip(f"not started, because {failed[0]} failed") + + +@pytest.fixture(scope="session", autouse=True) +def imswitch_reachable(): + """Without a server the whole suite skips.""" + if not client.reachable(): + pytest.skip(f"ImSwitch not reachable at {client.IMSWITCH_URL}") + + +@pytest.fixture(scope="session") +def live(request): + """``live("step 4/10 ...")`` prints a line at once, for a test that runs for minutes. + + pytest keeps what a test prints and shows it only when the test fails. + This switches that off for the one line. + """ + capture = request.config.pluginmanager.getplugin("capturemanager") + reporter = request.config.pluginmanager.getplugin("terminalreporter") + + def show(text): + with capture.global_and_fixture_disabled(): + reporter.ensure_newline() # pytest has written the test's name without a line end + reporter.write_line(text) + return show diff --git a/imswitch/imcontrol/_test/hil/firmware_sync.py b/imswitch/imcontrol/_test/hil/firmware_sync.py new file mode 100644 index 000000000..f93c37b36 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/firmware_sync.py @@ -0,0 +1,108 @@ +"""Bring every board to the firmware server's version. + +FLASHES FIRMWARE. + +A sync, not an update: a board with a newer developer build is flashed back as well. +The flashing itself is ImSwitch's own firmware update (startFirmwareUpdate), +which checks every image and counts a board as done only when it reports the new version. +""" +import time + +import requests + +import hil_client as client + +SCAN_TIMEOUT = 5 # seconds checkFirmwareUpdates waits for the boards on the CAN bus +SYNC_TIMEOUT = 1800 # seconds one firmware update may take + +# update_status values of a board that differs from the server and has an image to flash. +OUT_OF_SYNC = ("update_available", "device_newer") + + +class SyncFailed(Exception): + pass + + +def label(device): + return f"CAN {device.get('canId')} ({device.get('deviceTypeStr')}, {device['connection']})" + + +def scan_came_back_empty(result): + """True if the CAN scan found nothing: then even the master is listed without a canId.""" + usb = next((d for d in result.get("devices") or [] if d["connection"] == "usb"), None) + return (usb is not None and "can" in (usb.get("deviceTypeStr") or "").lower() + and usb.get("canId") is None) + + +def check(): + """checkFirmwareUpdates; asked up to three times, because the CAN scan sometimes is empty.""" + for _ in range(3): + result = client.get_json("/UC2ConfigController/checkFirmwareUpdates", + {"timeout": SCAN_TIMEOUT}, timeout=120) + if not scan_came_back_empty(result): + break + time.sleep(3) + else: + raise SyncFailed("the CAN scan came back empty three times") + if not result.get("server_version"): + raise SyncFailed(f"{result.get('firmware_server')} publishes no version.json") + return result + + +def out_of_sync(result, connection): + """The boards on "usb" (the master) or "can" (the nodes) that need flashing.""" + return [d for d in result["devices"] + if d["connection"] == connection and d["update_status"] in OUT_OF_SYNC] + + +def flash(can_ids, include_master): + """Start one ImSwitch firmware update and wait until it ended; raises unless it succeeded.""" + started = client.post_json("/UC2ConfigController/startFirmwareUpdate", json=can_ids, + params={"include_master": include_master}, timeout=120) + if started.get("status") != "started": + raise SyncFailed(f"ImSwitch refused the update: {started}") + + deadline = time.monotonic() + SYNC_TIMEOUT + status, message = {"state": "running"}, None + while status.get("state") == "running": + if time.monotonic() > deadline: + raise SyncFailed(f"the update still runs after {SYNC_TIMEOUT}s") + time.sleep(5) + try: + status = client.get_json("/UC2ConfigController/getFirmwareUpdateStatus", timeout=120) + except requests.RequestException: + continue # ImSwitch can be too busy flashing to answer; the update goes on + if status.get("message") != message: + message = status.get("message") + print(f"sync: {message}", flush=True) + + for step in status.get("steps") or []: + print(f"sync: {label(step)}: {step['status']} {step.get('message') or ''}", flush=True) + if status.get("homing_required"): + print("sync: the updated motors restarted and need homing", flush=True) + if status.get("state") != "success": + raise SyncFailed(f"the update ended {status.get('state')!r}") + + +def sync(): + """Flash every board that differs from the server; raises SyncFailed if one still does.""" + result = check() + print(f"sync: server version {result['server_version']}", flush=True) + for device in result["devices"]: + print(f"sync: {label(device)}: {device.get('installed_version') or 'no version'} " + f"[{device['update_status']}]", flush=True) + + # The master first, on its own: old master firmware cuts off the version the nodes report, + # so behind it no node would count as updated. + if out_of_sync(result, "usb"): + flash([], include_master=True) + result = check() + nodes = out_of_sync(result, "can") + if nodes: + flash([d["canId"] for d in nodes], include_master=False) + result = check() + + left = out_of_sync(result, "usb") + out_of_sync(result, "can") + if left: + raise SyncFailed("still out of sync: " + ", ".join(label(d) for d in left)) + print(f"sync: every board runs {result['server_version']}", flush=True) diff --git a/imswitch/imcontrol/_test/hil/hil_client.py b/imswitch/imcontrol/_test/hil/hil_client.py new file mode 100644 index 000000000..d4dc12e44 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/hil_client.py @@ -0,0 +1,102 @@ +"""HTTP access to the ImSwitch under test, and the setup it has loaded. + +An endpoint is written "/Controller/method", as in ImSwitch's API documentation: + + client.get_json("/PositionerController/getPositionerPositions") + client.get_json("/SettingsController/getCameraStatus", {"detectorName": "WidefieldCamera"}) + +What hardware the rig has is read from ImSwitch's own setup file: + + client.setup()["lasers"] # {"LED": {"valueRangeMax": 1023, ...}, ...} + +Only the server URL comes from the environment, so the suite can also run from a laptop. +""" +import functools +import os +import time + +import pytest +import requests + +IMSWITCH_URL = os.environ.get("IMSWITCH_URL", "http://localhost:8001").rstrip("/") + + +def get(endpoint, **kwargs): + """``requests.get`` on an endpoint; returns the response whatever its status.""" + kwargs.setdefault("timeout", 30) + return requests.get(f"{IMSWITCH_URL}/api{endpoint}", **kwargs) + + +def post(endpoint, **kwargs): + """``requests.post`` on an endpoint; returns the response whatever its status.""" + kwargs.setdefault("timeout", 30) + return requests.post(f"{IMSWITCH_URL}/api{endpoint}", **kwargs) + + +def json_of(response): + """The JSON of a response; any status but 200 fails the test.""" + assert response.status_code == 200, ( + f"{response.request.method} {response.url} -> {response.status_code}: {response.text}") + return response.json() + + +def get_json(endpoint, params=None, timeout=30): + return json_of(get(endpoint, params=params, timeout=timeout)) + + +def post_json(endpoint, params=None, json=None, timeout=30): + return json_of(post(endpoint, params=params, json=json, timeout=timeout)) + + +def reachable(): + try: + return get("/version", timeout=10).status_code == 200 + except requests.RequestException: + return False + + +@functools.lru_cache(maxsize=None) # asked once per run; an error is not cached +def setup_file(): + """The name of the setup file ImSwitch runs with, as imcontrol_options.json names it. + + The file is ImSwitchConfig/imcontrol_setups/; the name may also be a full path. + """ + return get_json("/UC2ConfigController/getCurrentSetupFilename")["current_setup"] + + +@functools.lru_cache(maxsize=None) +def setup(): + """The content of setup_file(), as a dict. + + ImSwitch parses the file before it answers. + So a field the file leaves out comes back with ImSwitch's default ("forAcquisition": false). + The tests use "detectors", "lasers", "LEDMatrixs", "positioners" and "objective". + A section the rig does not have is empty or None. + """ + return get_json("/UC2ConfigController/readSetupFile", {"setupFileName": setup_file()}) + + +def wait_until(check, timeout, interval=0.1): + """Call ``check()`` until it returns something true and return that; None after ``timeout``.""" + deadline = time.monotonic() + timeout + while True: + result = check() + if result or time.monotonic() >= deadline: + return result or None + time.sleep(interval) + + +def parametrize_from_rig(argnames, fetch, reason): + """``pytest.mark.parametrize(argnames, fetch())`` for values the rig reports, e.g. its axes. + + pytest reads the values while it collects the tests, where an error would stop the whole run. + So without a server, or with nothing to test, the test is skipped with ``reason``. + """ + try: + values = fetch() + except (requests.RequestException, AssertionError) as error: + values, reason = [], f"ImSwitch not reachable: {error}" + if not values: + nothing = [None] * len(argnames.split(",")) + values = [pytest.param(*nothing, id="none", marks=pytest.mark.skip(reason=reason))] + return pytest.mark.parametrize(argnames, values) diff --git a/imswitch/imcontrol/_test/hil/hil_pdf.py b/imswitch/imcontrol/_test/hil/hil_pdf.py new file mode 100644 index 000000000..4f21b1226 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/hil_pdf.py @@ -0,0 +1,232 @@ +"""hil-report.pdf: the results of the run, then the PDFs the tests appended. + +The first pages are drawn here with ReportLab. +They show what was tested and one line per test file. +Then comes each test file with what failed, its notes, its cases and its photos. +pypdf appends the PDFs of ``hil_report.append_pdf`` as they are. +""" +import io +import os +import time +from xml.sax.saxutils import escape + +from pypdf import PdfReader, PdfWriter +from reportlab.lib import colors +from reportlab.lib.pagesizes import A4 +from reportlab.lib.styles import ParagraphStyle +from reportlab.lib.utils import ImageReader +from reportlab.platypus import ( + Flowable, Image, Indenter, KeepTogether, Paragraph, SimpleDocTemplate, Spacer, Table, + TableStyle, +) +from reportlab.platypus.doctemplate import LayoutError + +from hil_report import by_file, name_of, verdict_of + +MARGIN = 42 +PADDING = 6 # points ReportLab's page frame keeps free inside the margins, on each side +WIDTH = A4[0] - 2 * (MARGIN + PADDING) # of the text +INDENT = 14 # of a test's part under its heading +INK = colors.HexColor("#24515e") # the headings +HEAD_ROW = colors.HexColor("#ececec") # behind the column names of a table +COLOUR = {"passed": colors.HexColor("#2e9c3f"), "failed": colors.HexColor("#d0312d"), + "error": colors.HexColor("#d0312d"), "skipped": colors.HexColor("#8a8a8a")} +OUTPUT_LINES = 12 # of a failure's traceback that go into the report +PHOTO_HEIGHT = 285 # points; a photo is scaled down to this + +BODY = ParagraphStyle("body", fontName="Times-Roman", fontSize=9, leading=11.5, + splitLongWords=True) +LABEL = ParagraphStyle("label", parent=BODY, fontName="Times-Bold") +CELLS = TableStyle([("VALIGN", (0, 0), (-1, -1), "TOP"), + ("LEFTPADDING", (0, 0), (-1, -1), 0), + ("TOPPADDING", (0, 0), (-1, -1), 1.5), + ("BOTTOMPADDING", (0, 0), (-1, -1), 1.5)]) + + +def draw_verdict(canvas, x, y, radius, status): + """A small square with round corners at (x, y): green with a tick, red with a cross, + grey with a dash.""" + canvas.setFillColor(COLOUR[status]) + canvas.roundRect(x - radius, y - radius, 2 * radius, 2 * radius, radius * 0.35, + stroke=0, fill=1) + canvas.setStrokeColor(colors.white) + canvas.setLineWidth(radius * 0.3) + d = radius * 0.45 + if status == "passed": + canvas.line(x - d, y, x - d * 0.3, y - d) + canvas.line(x - d * 0.3, y - d, x + d, y + d) + elif status == "skipped": + canvas.line(x - d, y, x + d, y) + else: + canvas.line(x - d, y - d, x + d, y + d) + canvas.line(x - d, y + d, x + d, y - d) + + +class Verdict(Flowable): + """The sign of draw_verdict, to put into a table.""" + + def __init__(self, status, size=9): + super().__init__() + self.status, self.size = status, size + + def wrap(self, available_width, available_height): + return self.size, self.size + + def draw(self): + draw_verdict(self.canv, self.size / 2, self.size / 2, self.size / 2, self.status) + + +class Heading(Flowable): + """A title on a thin line; with ``status``, its sign in front.""" + + def __init__(self, text, size, status=None): + super().__init__() + self.text, self.size, self.status = text, size, status + self.keepWithNext = True + self.spaceBefore, self.spaceAfter = size, 4 + + def wrap(self, available_width, available_height): + self.width = available_width + return available_width, self.size + 7 + + def draw(self): + x = 0 + if self.status: + radius = self.size * 0.5 + draw_verdict(self.canv, radius, 5 + self.size * 0.35, radius, self.status) + x = self.size + 5 + self.canv.setFillColor(INK) + self.canv.setFont("Helvetica-Bold", self.size) + self.canv.drawString(x, 5, self.text) + self.canv.setStrokeColor(INK) + self.canv.setLineWidth(0.5) + self.canv.line(0, 0, self.width, 0) + + +def text(value, style=BODY): + return Paragraph(escape(str(value)).replace("\n", "
"), style) + + +def rows(pairs, width): + """A table of (label, value): the label bold on the left, as in the head of the report.""" + table = Table([[text(label, LABEL), value if isinstance(value, Flowable) else text(value)] + for label, value in pairs], colWidths=[90, width - 90], hAlign="LEFT") + table.setStyle(CELLS) + return table + + +def grid(head, body, widths): + """A table with a line of column names, in bold on light grey.""" + table = Table([[text(name, LABEL) for name in head]] + body, colWidths=widths, repeatRows=1, + hAlign="LEFT") + table.setStyle(CELLS) + table.setStyle(TableStyle([("BACKGROUND", (0, 0), (-1, 0), HEAD_ROW), + ("LEFTPADDING", (0, 0), (0, -1), 3)])) + return table + + +def failure_lines(details): + """The lines of a pytest traceback that say what went wrong: its "E ..." lines.""" + lines = details.strip().splitlines() + errors = [line[1:].strip() for line in lines if line.startswith("E ")] + return (errors or lines)[-OUTPUT_LINES:] + + +def test_part(number, file, cases, description, notes, photos): + """The flowables of one test file: its heading, what it checks, output, cases and photos.""" + name = name_of(file) + width = WIDTH - INDENT + output = [] + for case, result in cases.items(): + if result["status"] in ("failed", "error"): + output.append(f"{case or name} {result['status']}:") + output.extend(failure_lines(result["details"])) + elif result["status"] == "skipped": + output.append(f"{case} skipped: {result['details']}") + output.extend(notes) + pairs = [("What it checks", description)] if description else [] + # One line per row, labelled once: a table splits onto the next page only between rows. + pairs += [("Output" if index == 0 else "", line) for index, line in enumerate(output)] + part = [Heading(f"{number}. {name}", 11, verdict_of([r["status"] for r in cases.values()])), + Indenter(left=INDENT)] + # A file without docstring that passed has nothing to say. + # ReportLab refuses a table without rows. + if pairs: + part.append(rows(pairs, width)) + if any(cases): + body = [[text(case), text(result["status"]), text(f"{result['duration']:.1f} s")] + for case, result in cases.items()] + part += [Spacer(1, 4), text("Cases", LABEL), + grid(["Case", "Result", "Time"], body, [width - 130, 70, 60])] + for case in cases: + for label, png in photos.get(f"{file}::{case}", []): + image_width, image_height = ImageReader(io.BytesIO(png)).getSize() + scale = min(width / image_width, PHOTO_HEIGHT / image_height) + part.append(KeepTogether([Spacer(1, 6), text(label), Image( + io.BytesIO(png), width=image_width * scale, height=image_height * scale, + hAlign="LEFT")])) + return part + [Indenter(left=-INDENT)] + + +def test_pages(setup, started, results, descriptions, photos, notes): + """The pages drawn here, as the bytes of a PDF.""" + files = by_file(results) + statuses = [result["status"] for result in results.values()] + counted = ", ".join(f"{statuses.count(status)} {status}" + for status in ("passed", "failed", "error", "skipped") + if status in statuses) + head = [("Started", time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(started))), + ("Duration", f"{(time.time() - started) / 60:.0f} min"), + ("Result", counted or "no test ran"), + ("Tested target", os.environ.get("HIL_TARGET")), + ("Run", os.environ.get("HIL_RUN_URL"))] + summary = [] + for number, (file, cases) in enumerate(files.items(), 1): + summary.append([text(number), Verdict(verdict_of([r["status"] for r in cases.values()])), + text(name_of(file)), text(descriptions.get(file, "")), + text(f"{sum(r['duration'] for r in cases.values()):.0f} s")]) + story = [Heading(f"Hardware test: {setup}", 14, verdict_of(statuses)), + rows([pair for pair in head if pair[1]], WIDTH), + Spacer(1, 6), text("Summary", LABEL), + grid(["No.", "Result", "Test", "What it checks", "Time"], summary, + [28, 42, 100, WIDTH - 215, 45])] + # The rig (os-rpi) passes the images of the pallet under test, one per line. + # One image per row: a table splits between its rows onto the next page, never inside a row. + images = os.environ.get("HIL_IMAGES", "").split() + if images: + story += [Spacer(1, 6), text("Images under test", LABEL), + grid(["Image"], [[text(image)] for image in images], [WIDTH])] + story.append(Heading("Tests", 13)) + for number, (file, cases) in enumerate(files.items(), 1): + story += test_part(number, file, cases, descriptions.get(file, ""), + notes.get(file, []), photos) + + def page_number(canvas, document): + canvas.setFont("Times-Roman", 8) + canvas.drawRightString(A4[0] - MARGIN, 24, str(document.page)) + + pages = io.BytesIO() + SimpleDocTemplate(pages, pagesize=A4, leftMargin=MARGIN, rightMargin=MARGIN, topMargin=MARGIN, + bottomMargin=MARGIN, title=f"Hardware test: {setup}").build( + story, onFirstPage=page_number, onLaterPages=page_number) + return pages.getvalue() + + +def write_pdf(target, setup, started, results, descriptions, photos, notes, appended): + """Write the report to the file ``target``. + + ``appended`` holds the (title, PDF bytes) the tests gave to ``hil_report.append_pdf``. + """ + try: + pages = test_pages(setup, started, results, descriptions, photos, notes) + except LayoutError as error: + raise ValueError(f"the report does not fit on its pages: {error}") from error + with PdfWriter() as writer: + writer.append(io.BytesIO(pages), outline_item="Hardware tests") + for title, data in appended: + writer.append(PdfReader(io.BytesIO(data)), outline_item=title) + writer.add_metadata({"/Title": f"Hardware test: {setup}", "/Author": "openUC2"}) + # Written under another name first, so a run that dies here leaves no half report. + pending = target.with_suffix(".pending") + writer.write(str(pending)) + pending.replace(target) diff --git a/imswitch/imcontrol/_test/hil/hil_report.py b/imswitch/imcontrol/_test/hil/hil_report.py new file mode 100644 index 000000000..0efd351f9 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/hil_report.py @@ -0,0 +1,196 @@ +"""The report of a run: hil-report.pdf (results, photos, PDFs of the tests) and summary.md. + +summary.md is the short form for the CI step summary: a table with one row per test file. + +A test adds to its part of the report with: + + attach("dark", image) a photo + note("The calibration ended as ...") a line of text + append_pdf("FRAME calibration", data) a whole PDF, after the pages of the tests + +``__main__.py`` passes ``ReportPlugin`` to pytest, +which collects every result and writes the two files at the end. +Under plain pytest nothing is written. +""" +import io +import os +import time +from pathlib import Path + +import numpy as np +from PIL import Image + +PHOTO_WIDTH = 640 # photos are scaled down to this, so a report with dozens of them stays small +STATUSES = ["failed", "error", "skipped", "passed"] # the order in the summary + +_started = time.time() +_results = {} # test id -> {"status", "duration", "details"} +_descriptions = {} # test file -> the first paragraph of its docstring: what it checks +_photos = {} # test id -> [(label, PNG bytes)] +_notes = {} # test file -> [lines of text] +_appended = [] # [(title, PDF bytes)] +report_error = None # why hil-report.pdf could not be written + + +def reset(): + global _started, report_error + _started = time.time() + report_error = None + _results.clear() + _descriptions.clear() + _photos.clear() + _notes.clear() + _appended.clear() + + +def current_test(): + """The id of the test that runs now, "test_x.py::test_y[cam]"; None outside pytest.""" + test = os.environ.get("PYTEST_CURRENT_TEST") # "test_x.py::test_y[cam] (call)" + return test.rsplit(" (", 1)[0] if test else None + + +def attach(label, image): + """Add a photo to the test that is running. ``image`` is a PIL image or a numpy array.""" + test = current_test() + if not test: + return + if isinstance(image, np.ndarray): + image = Image.fromarray(np.clip(image, 0, 255).astype(np.uint8)) + if image.mode not in ("L", "RGB"): + image = image.convert("RGB") + if image.width > PHOTO_WIDTH: + image = image.resize((PHOTO_WIDTH, round(image.height * PHOTO_WIDTH / image.width))) + buffer = io.BytesIO() + image.save(buffer, format="PNG", optimize=True) + _photos.setdefault(test, []).append((str(label), buffer.getvalue())) + + +def note(text): + """Add a line of text to the part of the test file that is running.""" + test = current_test() + if test: + _notes.setdefault(test.split("::")[0], []).append(str(text)) + + +def append_pdf(title, data): + """Append a PDF (its bytes) to the report, after the pages of the tests.""" + _appended.append((str(title), data)) + + +def record_problem(name, message): + """Add a problem outside the tests to the results, e.g. a camera that did not start.""" + _results[name] = {"status": "error", "duration": 0.0, "details": str(message)} + + +def counts(): + """``{"failed": 0, "error": 0, "skipped": 1, "passed": 29}``""" + statuses = [result["status"] for result in _results.values()] + return {status: statuses.count(status) for status in STATUSES} + + +def by_file(results): + """``{"test_board_link.py": {"test_uc2_board_connected": result, ...}, ...}``, in run order. + + A problem outside the tests (``record_problem``) is an entry without cases. + """ + files = {} + for test, result in results.items(): + file, _, case = test.partition("::") + files.setdefault(file, {})[case] = result + return files + + +def verdict_of(statuses): + """One status for several: failed when one failed, passed when one ran, else skipped.""" + if "failed" in statuses or "error" in statuses: + return "failed" + return "passed" if "passed" in statuses else "skipped" + + +def name_of(file): + """ "board_link" for "test_board_link.py"; a problem outside the tests keeps its name.""" + return file[len("test_"):-len(".py")] if file.endswith(".py") else file + + +def summary(setup_file): + """Markdown for the CI step summary: one row per test file, then one line per failure.""" + totals = " · ".join(f"{n} {status}" for status, n in counts().items()) + if report_error: + report = f"The PDF was not written: {report_error}" + else: + report = "Report: `hil-report.pdf` in the artifact `hil-report`." + lines = ["### Hardware tests", "", f"setup `{setup_file}` · {totals}", "", report, "", + "| No. | Test | Result | Cases | Time |", "|---|---|---|---|---|"] + for number, (file, cases) in enumerate(by_file(_results).items(), 1): + statuses = [result["status"] for result in cases.values()] + cases_counted = ", ".join(f"{statuses.count(status)} {status}" + for status in STATUSES if status in statuses) + verdict = verdict_of(statuses) + if verdict == "failed": + verdict = f"**{verdict}**" # bold, so a failure stands out in the table + seconds = sum(result["duration"] for result in cases.values()) + lines.append(f"| {number} | {name_of(file)} | {verdict} | {cases_counted} | " + f"{seconds:.0f} s |") + lines.append("") + for test, result in _results.items(): + if result["status"] in ("failed", "error"): + # The first "E ..." line of a pytest traceback is the assertion's message; + # a problem outside the tests has no such line, there the last line says it. + details = result["details"].strip().splitlines() or [""] + errors = [line[1:].strip() for line in details if line.startswith("E ")] + lines.append(f"- **{result['status']}** `{test}`: {(errors or details[-1:])[0][:300]}") + for file, notes in _notes.items(): + for text in notes: + lines.append(f"- `{file}`: {text}") + return "\n".join(lines) + "\n" + + +def write_reports(report_dir, setup_file): + """Write hil-report.pdf and summary.md. A PDF that fails is a problem in the summary.""" + global report_error + try: + # Imported here, so that a missing ReportLab also ends as a problem in the summary. + from hil_pdf import write_pdf + write_pdf(report_dir / "hil-report.pdf", setup_file, _started, _results, _descriptions, + _photos, _notes, _appended) + except Exception as error: + report_error = str(error) + record_problem("PDF generation", report_error) + print(f"[hil] PDF generation failed: {error}", flush=True) + (report_dir / "summary.md").write_text(summary(setup_file), encoding="utf-8") + + +class ReportPlugin: + """Collects the results while pytest runs and writes the report files into ``report_dir``. + + ``setup_file`` is the ImSwitch setup the rig ran with; both files name it at the top. + """ + + def __init__(self, report_dir, setup_file): + self.report_dir = Path(report_dir) + self.setup_file = setup_file + + def pytest_itemcollected(self, item): + """Called for every test pytest finds: keep what its file says it checks. + + That is the first paragraph of the file's docstring, its lines joined into one. + """ + file = item.nodeid.split("::")[0] + first_paragraph = (item.module.__doc__ or "").strip().split("\n\n")[0] + _descriptions[file] = " ".join(first_paragraph.split()) + + def pytest_runtest_logreport(self, report): + """Called three times per test: for its setup, the test itself (call) and its teardown.""" + result = _results.setdefault(report.nodeid, { + "status": "passed", "duration": 0.0, "details": ""}) + result["duration"] += report.duration + if report.failed and result["status"] != "failed": + # A failure outside the test itself is a broken fixture, not a failed assertion. + result["status"] = "failed" if report.when == "call" else "error" + result["details"] = str(report.longrepr) + elif report.skipped and result["status"] == "passed": + result["status"] = "skipped" + result["details"] = report.longrepr[2] # (file, line, reason) + + def pytest_sessionfinish(self): + write_reports(self.report_dir, self.setup_file) diff --git a/imswitch/imcontrol/_test/hil/pytest.ini b/imswitch/imcontrol/_test/hil/pytest.ini new file mode 100644 index 000000000..a7ab0175d --- /dev/null +++ b/imswitch/imcontrol/_test/hil/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +# The HIL suite is its own pytest project, so pytest does not import the ImSwitch packages above. +# console_output_style = count shows "[12/31]" after each test, the number of tests done. +# The default is a percentage, which reads as time. +# That misleads here: the calibration is the last 2 tests and most of the run. +addopts = -p no:arkitekt_next -p no:cacheprovider -v --tb=short +console_output_style = count +markers = + hardware: needs a real rig; conftest.py sets it on every HIL test diff --git a/imswitch/imcontrol/_test/hil/test_board_link.py b/imswitch/imcontrol/_test/hil/test_board_link.py new file mode 100644 index 000000000..09eb90b72 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_board_link.py @@ -0,0 +1,19 @@ +"""ImSwitch talks to the UC2 board over USB, and the board runs the master firmware, +the one that passes commands on to the motor and LED boards over the CAN bus. + +Only reads. +""" +import hil_client as client + + +def test_uc2_board_connected(): + # Not is_connected: that one only says that ImSwitch answers, it is always true. + connected = client.get_json("/UC2ConfigController/uc2_board_is_connected", timeout=5) + assert connected is True, "UC2/ESP32 board is not reported as connected" + + +def test_correct_esp32_firmware(): + info = client.get_json("/UC2ConfigController/getFirmwareInfo", timeout=5) + print(f"\nESP32 firmware: {info}") + assert info.get("connected") is True, f"ESP32 not connected: {info}" + assert info.get("isMaster") is True, f"ESP32 does not run the master firmware: {info}" diff --git a/imswitch/imcontrol/_test/hil/test_camera_capture.py b/imswitch/imcontrol/_test/hil/test_camera_capture.py new file mode 100644 index 000000000..2beff4835 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_camera_capture.py @@ -0,0 +1,82 @@ +"""Each camera takes a picture, and the picture is as large as the camera says its sensor is. + +Only reads. + +ImSwitch reads the two kinds of camera in different ways: +- An acquisition detector ("forAcquisition": true in the setup) with snapNumpyToFastAPI. + That only works while its live view runs; the runner (__main__.py) has started it. +- The observation camera ("forAcquisition": false) with snapOverviewImage, at any time. +""" +import base64 +import io + +import pytest +from PIL import Image + +import hil_client as client +from hil_report import attach + +SNAP_RESIZE = 0.1 # resizeFactor of the acquisition frame, to keep the transfer small + + +def detectors(): + return list(client.setup()["detectors"]) + + +def is_observation_camera(detector): + return not client.setup()["detectors"][detector]["forAcquisition"] + + +def camera_status(detector): + """getCameraStatus; fails the test if the camera has an error or is ImSwitch's mock camera. + + The setup names the camera, so a mock in its place means the real one did not start. + """ + status = client.get_json("/SettingsController/getCameraStatus", {"detectorName": detector}) + assert "error" not in status and status.get("status") != "error", ( + f"{detector}: getCameraStatus failed: {status.get('error')}") + assert not status.get("isMock") and str(status.get("model", "")).lower() != "mock", ( + f"{detector}: ImSwitch served a mock camera (model={status.get('model')!r})") + return status + + +def snap(detector): + """One frame as (PIL image, the resize factor ImSwitch applied to it).""" + if is_observation_camera(detector): + # camera_name only names the folder ImSwitch saves the frame to. + body = client.post_json("/ExperimentController/snapOverviewImage", timeout=10, + params={"slot_id": "1", "camera_name": "hil_camera_capture"}) + frame = Image.open(io.BytesIO(base64.b64decode(body["imageBase64"]))) + assert frame.format == "JPEG", f"snapOverviewImage returned {frame.format}" + return frame, 1.0 + response = client.get("/RecordingController/snapNumpyToFastAPI", timeout=10, params={ + "detectorName": detector, "resizeFactor": SNAP_RESIZE}) + assert response.status_code == 200, response.text + frame = Image.open(io.BytesIO(response.content)) + assert frame.format == "PNG", f"snapNumpyToFastAPI returned {frame.format}" + return frame, SNAP_RESIZE + + +every_detector = client.parametrize_from_rig("detector", detectors, "the setup has no detectors") + + +@every_detector +def test_camera_returns_image(detector): + camera_status(detector) + frame, _ = snap(detector) + attach(detector, frame) + assert frame.width > 0 and frame.height > 0 + + +@every_detector +def test_camera_matches_sensor_resolution(detector): + status = camera_status(detector) + width = status.get("currentWidth") or status.get("sensorWidth") + height = status.get("currentHeight") or status.get("sensorHeight") + if not width or not height: + pytest.skip(f"{detector}: getCameraStatus reports no frame size ({width}x{height})") + frame, resize = snap(detector) + expected = (int(width * resize), int(height * resize)) + assert frame.size == expected, ( + f"{detector}: got {frame.width}x{frame.height}, expected {expected[0]}x{expected[1]} " + f"({width}x{height} at resizeFactor {resize})") diff --git a/imswitch/imcontrol/_test/hil/test_firmware_server.py b/imswitch/imcontrol/_test/hil/test_firmware_server.py new file mode 100644 index 000000000..c2ceac5dc --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_firmware_server.py @@ -0,0 +1,103 @@ +"""The firmware server can be reached and has firmware files on it, +and each board on the CAN bus runs the version the server has for it. + +Only reads, flashes nothing. + +checkFirmwareUpdates is how ImSwitch compares the server's version.json with each board. +To flash before the tests, start the suite with --sync-firmware. +""" +import time + +import pytest + +import hil_client as client + +SCAN_TIMEOUT = 5 # seconds checkFirmwareUpdates waits for the boards on the CAN bus +# update_status values of a board that differs from the server and has an image to flash. +OUT_OF_SYNC = ("update_available", "device_newer") + + +def label(device): + return f"CAN {device.get('canId')} ({device.get('deviceTypeStr')}, {device['connection']})" + + +def scan_came_back_empty(result): + """True if the CAN scan found nothing: then even the master is listed without a canId.""" + usb = next((d for d in result.get("devices") or [] if d["connection"] == "usb"), None) + return (usb is not None and "can" in (usb.get("deviceTypeStr") or "").lower() + and usb.get("canId") is None) + + +@pytest.fixture(scope="module") +def check(): + """The answer of checkFirmwareUpdates, kept for the file: every call scans the CAN bus. + + Asked up to three times, because the CAN scan sometimes is empty. + """ + for _ in range(3): + result = client.get_json("/UC2ConfigController/checkFirmwareUpdates", + {"timeout": SCAN_TIMEOUT}, timeout=60) + if not scan_came_back_empty(result): + break + time.sleep(3) + else: + pytest.fail("the CAN scan came back empty three times: " + "no node listed, the master without canId") + print() + for device in result["devices"]: + print(f"{label(device)}: {device.get('installed_version') or 'no version'}, " + f"image {device.get('filename') or 'none'} [{device['update_status']}]") + return result + + +@pytest.fixture(scope="module") +def answering(check): + """The boards that answered the scan. The others may just be powered down.""" + boards = [d for d in check["devices"] if d["update_status"] != "unreachable"] + if not boards: + pytest.skip("no board answered") + return boards + + +def test_master_firmware_is_reported(): + info = client.get_json("/UC2ConfigController/getFirmwareInfo", timeout=60) + if info.get("status") == "error": + pytest.skip(f"getFirmwareInfo failed: {info.get('message')}") + print(f"\nmaster: {info}") + assert info.get("connected") is True, f"master board is not connected: {info}" + for field in ("name", "version", "date", "pindef"): + assert info.get(field), f"master firmware reports no {field}: {info}" + + +def test_firmware_server_is_configured(): + url = client.get_json("/UC2ConfigController/getOTAFirmwareServer").get("firmware_server_url") + print(f"\nfirmware server: {url}") + assert url, "no OTA firmware server configured" + + +def test_firmware_server_lists_binaries(): + listing = client.get_json("/UC2ConfigController/listAllFirmwareFiles", timeout=60) + if listing.get("status") != "success": + pytest.skip(f"firmware server unusable: {listing.get('message')}") + files = listing.get("files") or [] + print(f"\n{len(files)} firmware files on {listing.get('firmware_server')}") + assert files, f"firmware server {listing.get('firmware_server')} offers no .bin files" + for entry in files: + assert entry.get("filename", "").endswith(".bin"), entry + assert entry.get("url"), entry + + +def test_answering_boards_have_firmware(check, answering): + missing = [label(d) for d in answering + if d["update_status"] == "no_firmware" or not d.get("filename")] + assert not missing, f"no image on {check.get('firmware_server')} for: {', '.join(missing)}" + + +def test_boards_run_server_version(check, answering): + """Exactly the server's version: a board with a newer build fails too.""" + server = check.get("server_version") + if not server: + pytest.skip(f"{check.get('firmware_server')} names no version (no version.json?)") + stale = [f"{label(d)} runs {d.get('installed_version')}" + for d in answering if d["update_status"] in OUT_OF_SYNC] + assert not stale, f"not on the server's version {server}: {', '.join(stale)}" diff --git a/imswitch/imcontrol/_test/hil/test_frame_calibration.py b/imswitch/imcontrol/_test/hil/test_frame_calibration.py new file mode 100644 index 000000000..0ea5ff801 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_frame_calibration.py @@ -0,0 +1,246 @@ +"""The FRAME calibration runs from start to end with the calibration target in the holder, +and none of its steps fails. + +MOVES REAL HARDWARE: homes X, Y, Z and the turret, scans the target, focuses, changes objective. +SWITCHES LIGHT. +WRITES THE SETUP: the calibration stores its result in the setup file the rig runs with. +The result is the stage offset, the objective offsets and the pixel calibration. +It stays, it is what the calibration is for. +To go back by hand, the old setup is saved in the report folder +as setup_before_frame_calibration.json. + +The calibration target (plate or pinhole slides, setting "target") must be in the holder. +Afterwards the settings this test changed are back and every light is off. +""" +import io +import json +import os +import time +from pathlib import Path + +import pytest +from pypdf import PdfReader + +import hil_client as client +import hil_report + +RUN_TIMEOUT = 1800 # seconds the whole calibration may take +POLL_S = 2 # seconds between two status requests +STOP_TIMEOUT = 60 # seconds a stopped calibration may take to end +# Without a focus the calibration would wait for someone to focus by hand and press Continue. +# Nobody is there in a test, so the run fails instead. +# The light value is set, not kept ("illuminationValue" 0 would keep its current value). +# The teardown turns every light to 0, so a kept value would be 0 in the next run: a black camera. +TEST_SETTINGS = {"manualFocusFallback": False, "illuminationValue": 500} +BACKUP_NAME = "setup_before_frame_calibration.json" +CALIBRATION_PDF = "frame_calibration.pdf" # the calibration's own report, in the report folder +ENDED = ("finished", "failed", "aborted") +ACTIVE = ("running", "waiting") # the states the FRAME calibration app shows as running + + +# --- the calibration -------------------------------------------------------------------------- + +def status(): + return client.get_json("/FrameCalibrationController/getFrameCalibrationStatus") + + +def set_settings(values): + return client.post_json("/FrameCalibrationController/setFrameCalibrationSettings", + json=values) + + +def running(): + """Running as the app shows it, from the state of the status. + + The state ends before the calibration thread does. + The thread then only resets the camera and writes its report folder. + It switches no light, moves no motor and leaves the setup file alone. + So nothing this test does afterwards collides with it. + """ + return status()["state"] in ACTIVE + + +def stop_and_wait(): + if not running(): + return + client.post_json("/FrameCalibrationController/stopFrameCalibration") + assert client.wait_until(lambda: not running(), STOP_TIMEOUT, interval=POLL_S), ( + f"calibration thread did not stop within {STOP_TIMEOUT}s") + + +def collect_report(run): + """Put the calibration's own PDF into the report; never the one of an older run.""" + assert not running(), "cannot export while calibration still runs" + current = status() + assert run.get("startedAt") and current["startedAt"] == run["startedAt"], ( + "calibration identity changed; refusing to attach another run's report") + hil_report.note(f"The calibration ended as {current['state']}: {current['message']}") + response = client.get("/FrameCalibrationController/getFrameCalibrationReportPdf", timeout=120) + assert response.status_code == 200 and response.content.startswith(b"%PDF-"), ( + f"calibration PDF export failed: HTTP {response.status_code}") + # Checked again, because the endpoint gives the report of whatever run is the last one now. + assert status()["startedAt"] == run["startedAt"], "calibration changed during PDF export" + assert len(PdfReader(io.BytesIO(response.content)).pages), "calibration PDF has no pages" + hil_report.append_pdf("FRAME calibration (original report)", response.content) + # Also on its own, so it is in the report folder even when hil-report.pdf cannot be written. + folder = Path(os.environ.get("HIL_REPORT_DIR", ".")) + folder.mkdir(parents=True, exist_ok=True) + (folder / CALIBRATION_PDF).write_bytes(response.content) + hil_report.note(f"Its own report follows on the last pages, and is saved as {CALIBRATION_PDF}.") + + +def show_progress(steps, shown, live): + """Print what changed since the last call: a line when a step starts, one when it ends. + + calibration 4/10 First objective: find via, focus, affine ... + calibration 4/10 ok (212 s) objective 0: 0.598 x 0.598 um/px + + ``shown`` remembers the lines already printed, as (step name, "start" or "end"). + """ + for number, step in enumerate(steps, 1): + head = f"calibration {number:>2}/{len(steps)} " + if step["status"] != "pending" and (step["name"], "start") not in shown: + shown.add((step["name"], "start")) + live(f"{head}{step['title']} ...") + if step["status"] not in ("pending", "running") and (step["name"], "end") not in shown: + shown.add((step["name"], "end")) + took = f" ({step['seconds']:.0f} s)" if step["seconds"] is not None else "" + live(f"{head}{step['status']}{took} {step['detail']}") + for warning in step["warnings"]: + live(f"{' ' * len(head)}warning: {warning}") + + +def run_calibration(run, live): + """Start the calibration and follow it to its end; returns the last status. + + ``live`` prints the progress at once (conftest.py), so the log shows which step runs. + """ + started = client.post_json("/FrameCalibrationController/startFrameCalibration") + assert started.get("success"), f"startFrameCalibration refused: {started}" + run["accepted"] = True + run["startedAt"] = status()["startedAt"] + shown = set() + deadline = time.monotonic() + RUN_TIMEOUT + while True: + current = status() + show_progress(current["steps"], shown, live) + assert current["startedAt"] == run["startedAt"], "calibration replaced during polling" + if current["state"] in ENDED: + return current + # The test turned the manual focus off; a wait for the user would never end. + assert current["state"] != "waiting", ( + f"the calibration waits for the user: {current['message']}") + assert time.monotonic() < deadline, ( + f"the calibration did not end within {RUN_TIMEOUT}s (step {current['step']!r})") + time.sleep(POLL_S) + + +# --- the rig before and after ----------------------------------------------------------------- + +def all_lights_off(): + """Every laser, LED and the LED matrix off. One that fails does not stop the others.""" + actions = [] + for name in client.setup()["lasers"]: + actions.extend([ + ("/LaserController/setLaserValue", {"laserName": name, "value": 0}), + ("/LaserController/setLaserActive", {"laserName": name, "active": False}), + ]) + if client.setup()["LEDMatrixs"]: + actions.append(("/LEDMatrixController/setAllLEDOff", None)) + errors = [] + for endpoint, params in actions: + try: + client.get_json(endpoint, params) + except Exception as error: + errors.append(f"{endpoint} {params}: {error}") + if errors: + raise RuntimeError("; ".join(errors)) + + +def save_setup_backup(): + """Write the setup as it is now into the report folder.""" + folder = Path(os.environ.get("HIL_REPORT_DIR", ".")) + folder.mkdir(parents=True, exist_ok=True) + file = folder / BACKUP_NAME + backup = {"setupFile": client.setup_file(), "setup": client.setup()} + file.write_text(json.dumps(backup, indent=2), encoding="utf-8") + print(f"\nsetup before the calibration saved to {file.resolve()}") + hil_report.note("The calibration writes its result into the setup. The setup as it was " + f"before is saved as {BACKUP_NAME}.") + + +# pytest ends these fixtures in the reverse order of the arguments of ``calibration``: +# first the light goes off, then the settings go back, then the report is collected. +# Each of them runs also when the one before it failed. + +@pytest.fixture(scope="module") +def run(): + """What is known of the calibration this test starts: "accepted" and "startedAt". + + Afterwards its PDF goes into the report, also when the calibration failed. + """ + run = {} + yield run + if run.get("accepted"): + collect_report(run) + + +@pytest.fixture(scope="module") +def settings_for_the_test(): + """Skip when there is nothing to calibrate; else set TEST_SETTINGS, and put them back.""" + if not client.setup()["positioners"]: + pytest.skip("the setup has no stage") + response = client.get("/FrameCalibrationController/getFrameCalibrationSettings") + if response.status_code == 404: + pytest.skip("this ImSwitch has no FrameCalibrationController") + original = client.json_of(response) + if running(): + pytest.skip(f"a FRAME calibration is already running: {status()['state']}") + + save_setup_backup() + print(f"target {original['target']!r}, settings changed for the test: {TEST_SETTINGS}") + set_settings(TEST_SETTINGS) + yield + set_settings({key: original[key] for key in TEST_SETTINGS}) + + +@pytest.fixture(scope="module") +def lights_off_afterwards(): + """The calibration leaves its light on when it ends.""" + yield + all_lights_off() + + +@pytest.fixture(scope="module") +def calibration(live, run, settings_for_the_test, lights_off_afterwards): + """Run the calibration once; returns its last status.""" + try: + result = run_calibration(run, live) + finally: + if run.get("accepted"): + stop_and_wait() + live(f"calibration {result['state']}: {result['message']}") + return result + + +# --- tests ------------------------------------------------------------------------------------ + +def test_calibration_finishes(calibration): + assert calibration["state"] == "finished", ( + f"the calibration ended as {calibration['state']!r} in step {calibration['step']!r}: " + f"{calibration['message']} - is the calibration target in the holder?") + + +def test_every_step_succeeds(calibration): + """Optional steps may fail without stopping the calibration; this test lists them.""" + # Notes, not prints: a print only shows when the test fails. + for step in calibration["steps"]: + if step["warnings"]: + hil_report.note(f"Warning in {step['title']}: {'; '.join(step['warnings'])}") + # A skipped step did not test its part of the rig, e.g. the turret without a second objective. + skipped = [step["title"] for step in calibration["steps"] if step["status"] == "skipped"] + if skipped: + hil_report.note(f"Skipped by the calibration, so not tested: {', '.join(skipped)}") + failed = [f"{step['title']}: {step['detail']}" for step in calibration["steps"] + if step["status"] in ("failed", "aborted")] + assert not failed, "steps that failed:\n" + "\n".join(failed) diff --git a/imswitch/imcontrol/_test/hil/test_ledmatrix.py b/imswitch/imcontrol/_test/hil/test_ledmatrix.py new file mode 100644 index 000000000..049ce695d --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_ledmatrix.py @@ -0,0 +1,253 @@ +"""The LED matrix can be switched on and off, and every camera sees it go on and off. + +SWITCHES LIGHT. + +It passes only when every camera of the setup sees both switches. +On: the image changes by more than the camera's own noise. +Off: the image is closer to dark than to the lit image. +Only differences between frames are judged, never a brightness by itself. +The noise: take a few dark frames and measure how much they differ. +""" +import base64 +import contextlib +import io +import time + +import pytest +from PIL import Image, ImageChops, ImageStat + +import hil_client as client +from hil_report import attach, note + +MATRIX_INTENSITY = 255 # per colour, of setAllLED (0..255) +# Wait after switching a light, before the next frame. +# The observation camera sets its exposure by itself and needs more than a second for it. +LIGHT_SETTLE_S = 1.5 +AUTO_EXPOSURE_RESET_MS = 1500 # ImSwitch returns to manual exposure after this time +FRAME_RESIZE = 0.25 # the frames are compared at this share of their size +# The matrix must change the image by max(MIN_CHANGE, noise * NOISE_FACTOR) grey values. +# The noise is the largest difference between NOISE_SAMPLES dark frames. +MIN_CHANGE = 0.5 +NOISE_FACTOR = 1.5 +NOISE_SAMPLES = 6 +NOISE_DELAY_S = 0.5 + + +# --- lights ----------------------------------------------------------------------------------- + +def all_lights_off(): + """Every laser, LED and the LED matrix off, so only the matrix changes the image.""" + for name in client.setup()["lasers"]: + client.get_json("/LaserController/setLaserValue", {"laserName": name, "value": 0}) + client.get_json("/LaserController/setLaserActive", {"laserName": name, "active": False}) + client.get_json("/LEDMatrixController/setAllLEDOff") + time.sleep(LIGHT_SETTLE_S) + + +def set_matrix(on): + """Every LED of the matrix on at MATRIX_INTENSITY, or off.""" + if on: + client.get_json("/LEDMatrixController/setAllLED", { + "intensity_r": MATRIX_INTENSITY, "intensity_g": MATRIX_INTENSITY, + "intensity_b": MATRIX_INTENSITY}) + else: + client.get_json("/LEDMatrixController/setAllLEDOff") + time.sleep(LIGHT_SETTLE_S) + + +# --- cameras ---------------------------------------------------------------------------------- +# An acquisition detector ("forAcquisition": true) gives frames only while its live view runs. +# The observation camera ("forAcquisition": false) gives them at any time. + +class LeftOut(Exception): + """A camera that cannot give frames now; the test leaves it out and names it in the report.""" + + +def is_observation_camera(detector): + return not client.setup()["detectors"][detector]["forAcquisition"] + + +def cameras(): + """The cameras that give real frames. + A camera ImSwitch serves as a mock, or cannot report on, is left out and named in the report. + """ + found = [] + for detector in client.setup()["detectors"]: + status = client.get_json("/SettingsController/getCameraStatus", {"detectorName": detector}) + if "error" in status or status.get("status") == "error": + note(f"{detector} left out: getCameraStatus failed: {status.get('error')}") + elif status.get("isMock") or str(status.get("model", "")).lower() == "mock": + note(f"{detector} left out: ImSwitch served a mock camera") + else: + found.append(detector) + return found + + +def take_image(detector): + """One small greyscale frame (PIL) of either kind of camera.""" + if is_observation_camera(detector): + # camera_name only names the folder ImSwitch saves the frame to. + response = client.post("/ExperimentController/snapOverviewImage", timeout=60, + params={"slot_id": "1", "camera_name": "hil_ledmatrix"}) + if response.status_code == 400: + raise LeftOut(f"observation camera not connected: {response.text}") + raw = base64.b64decode(client.json_of(response)["imageBase64"]) + frame = Image.open(io.BytesIO(raw)).convert("L") + return frame.resize((max(1, int(frame.width * FRAME_RESIZE)), + max(1, int(frame.height * FRAME_RESIZE)))) + response = client.get("/RecordingController/snapNumpyToFastAPI", timeout=60, + params={"detectorName": detector, "resizeFactor": FRAME_RESIZE}) + assert response.status_code == 200, response.text + return Image.open(io.BytesIO(response.content)).convert("L") + + +def image_difference(first, second): + """Mean absolute grey-value difference between two frames.""" + return ImageStat.Stat(ImageChops.difference(first, second)).mean[0] + + +def brightness(image): + """Mean grey value of a frame.""" + return ImageStat.Stat(image).mean[0] + + +def exposure_and_gain(detector): + """ "exposure 20, gain 0" as the camera reports them; None for a value it does not report.""" + values = [client.get_json("/SettingsController/getDetectorParameter", + {"detectorName": detector, "parameterName": name}) + for name in ("exposure", "gain")] + return f"exposure {values[0]}, gain {values[1]}" + + +def dark_baseline(detector): + """Measure the camera noise with the scene unchanged. + + Returns (last frame, noise floor, change a light must exceed). + """ + previous = take_image(detector) + drifts = [] + for _ in range(NOISE_SAMPLES - 1): + time.sleep(NOISE_DELAY_S) + current = take_image(detector) + drifts.append(image_difference(previous, current)) + previous = current + noise_floor = max(drifts) + required = max(MIN_CHANGE, noise_floor * NOISE_FACTOR) + print(f"\n{detector} dark noise: drifts={[round(d, 2) for d in drifts]}, " + f"noise_floor={noise_floor:.2f}, required_change={required:.2f}") + return previous, noise_floor, required + + +@contextlib.contextmanager +def live_view(detector): + """Keep an acquisition detector streaming inside a ``with`` block. + + Only a live view that was started here is stopped again. + """ + # The HTTP status is 200 even when ImSwitch refuses, so only the "status" field tells. + status = client.post_json("/LiveViewController/startLiveView", {"detectorName": detector}, + timeout=60).get("status") + if status == "long_exposure": + raise LeftOut("exposure too long for live view") + assert status in ("success", "already_running"), ( + f"{detector}: startLiveView answered {status!r}") + time.sleep(0.4) + try: + yield + finally: + if status == "success": + client.get_json("/LiveViewController/stopLiveView", {"detectorName": detector}) + + +def auto_expose(detector): + """One auto-exposure pass on the scene as it is now; needs a running live view.""" + client.get_json("/SettingsController/setDetectorExposureOnce", + {"detectorName": detector, "resetDelayMs": AUTO_EXPOSURE_RESET_MS}) + time.sleep(AUTO_EXPOSURE_RESET_MS / 1000 + 0.1) + + +@contextlib.contextmanager +def ready(detector): + """The camera ready to take frames inside a ``with`` block. + + An acquisition detector streams, and its exposure is set once, in the dark, + so that the dark, the lit and the switched-off frames share it. + The observation camera is left alone: it has no live view for the auto exposure to act on. + """ + if is_observation_camera(detector): + yield + return + with live_view(detector): + auto_expose(detector) + yield + + +def sees_switching(detector, switch): + """Switch the light on and off in front of one camera; ``switch(on)`` switches it. + + Returns what the camera did not see, as lines; empty when it saw both. + """ + with ready(detector): + dark, noise_floor, required = dark_baseline(detector) + switch(True) + lit = take_image(detector) + switch(False) + off = take_image(detector) + settings = exposure_and_gain(detector) + attach(f"{detector}: dark", dark) + attach(f"{detector}: on", lit) + attach(f"{detector}: off again", off) + on_change, off_change = image_difference(dark, lit), image_difference(dark, off) + print(f"\n{detector}: on changed {on_change:.2f}, off is {off_change:.2f} from dark, " + f"noise floor {noise_floor:.2f}, required {required:.2f}; brightness dark " + f"{brightness(dark):.2f}, on {brightness(lit):.2f}; {settings}") + missed = [] + if on_change < required: + missed.append(f"{detector}: on changed the image only by {on_change:.2f}, " + f"need >= {required:.2f} (noise floor {noise_floor:.2f}; {settings})") + # Off is judged against on, not against the noise: + # the observation camera's own exposure drifts after a bright light. + if off_change >= on_change / 2: + missed.append(f"{detector}: off is still {off_change:.2f} from dark, " + f"need < {on_change / 2:.2f} (half of what on changed)") + return missed + + +# --- fixtures --------------------------------------------------------------------------------- + +@pytest.fixture(scope="module", autouse=True) +def led_matrix(): + if not client.setup()["LEDMatrixs"]: + pytest.skip("the setup has no LED matrix") + + +@pytest.fixture +def dark(): + """Every light off before and after the test.""" + all_lights_off() + yield + all_lights_off() + + +# --- tests ------------------------------------------------------------------------------------ + +def test_all_leds_off_is_accepted(): + client.get_json("/LEDMatrixController/setAllLEDOff", timeout=5) + + +def test_every_camera_sees_the_matrix_switch_on_and_off(dark): + found = cameras() + if not found: + pytest.skip("the setup has no camera that gives real frames") + missed, measured = [], 0 + # Every camera is measured before the verdict, so the report has the photos of all of them. + for detector in found: + try: + missed += sees_switching(detector, set_matrix) + measured += 1 + except LeftOut as reason: + note(f"{detector} left out: {reason}") + all_lights_off() + assert not missed, f"LED matrix @ {MATRIX_INTENSITY}:\n" + "\n".join(missed) + if not measured: + pytest.skip("no camera of the setup could give frames") diff --git a/imswitch/imcontrol/_test/hil/test_lightsource.py b/imswitch/imcontrol/_test/hil/test_lightsource.py new file mode 100644 index 000000000..883566072 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_lightsource.py @@ -0,0 +1,274 @@ +"""Each laser and LED can be switched on and off, and every camera sees each LED go on and off. + +SWITCHES LIGHT. + +One case per LED; it passes only when every camera of the setup sees both switches. +On: the image changes by more than the camera's own noise. +Off: the image is closer to dark than to the lit image. +Only differences between frames are judged, never a brightness by itself. +The noise: take a few dark frames and measure how much they differ. +""" +import base64 +import contextlib +import io +import time + +import pytest +from PIL import Image, ImageChops, ImageStat + +import hil_client as client +from hil_report import attach, note + +# Wait after switching a light, before the next frame. +# The observation camera sets its exposure by itself and needs more than a second for it. +LIGHT_SETTLE_S = 1.5 +AUTO_EXPOSURE_RESET_MS = 1500 # ImSwitch returns to manual exposure after this time +FRAME_RESIZE = 0.25 # the frames are compared at this share of their size +# A light must change the image by max(MIN_CHANGE, noise * NOISE_FACTOR) grey values. +# The noise is the largest difference between NOISE_SAMPLES dark frames. +MIN_CHANGE = 0.5 +NOISE_FACTOR = 1.5 +NOISE_SAMPLES = 6 +NOISE_DELAY_S = 0.5 + + +# --- lights ----------------------------------------------------------------------------------- + +def light_sources(): + """Names of every laser and LED of the setup.""" + return list(client.setup()["lasers"]) + + +def leds(): + """Names of the LEDs. + The lasers are left out: on these rigs their beam misses the sensor. + + The setup lists lasers and LEDs alike under "lasers", so only the name tells them apart. + """ + return [name for name in light_sources() + if "led" in name.lower() and "laser" not in name.lower()] + + +def set_light(name, on): + """Switch one laser/LED on (at the highest value its setup allows) or off.""" + value = client.setup()["lasers"][name]["valueRangeMax"] if on else 0 + client.get_json("/LaserController/setLaserValue", {"laserName": name, "value": value}) + client.get_json("/LaserController/setLaserActive", {"laserName": name, "active": on}) + time.sleep(LIGHT_SETTLE_S) + + +def all_lights_off(): + """Every laser, LED and the LED matrix off, so only the light under test changes the image.""" + for name in light_sources(): + client.get_json("/LaserController/setLaserValue", {"laserName": name, "value": 0}) + client.get_json("/LaserController/setLaserActive", {"laserName": name, "active": False}) + if client.setup()["LEDMatrixs"]: + client.get_json("/LEDMatrixController/setAllLEDOff") + time.sleep(LIGHT_SETTLE_S) + + +# --- cameras ---------------------------------------------------------------------------------- +# An acquisition detector ("forAcquisition": true) gives frames only while its live view runs. +# The observation camera ("forAcquisition": false) gives them at any time. + +class LeftOut(Exception): + """A camera that cannot give frames now; the test leaves it out and names it in the report.""" + + +def is_observation_camera(detector): + return not client.setup()["detectors"][detector]["forAcquisition"] + + +def cameras(): + """The cameras that give real frames. + A camera ImSwitch serves as a mock, or cannot report on, is left out and named in the report. + """ + found = [] + for detector in client.setup()["detectors"]: + status = client.get_json("/SettingsController/getCameraStatus", {"detectorName": detector}) + if "error" in status or status.get("status") == "error": + note(f"{detector} left out: getCameraStatus failed: {status.get('error')}") + elif status.get("isMock") or str(status.get("model", "")).lower() == "mock": + note(f"{detector} left out: ImSwitch served a mock camera") + else: + found.append(detector) + return found + + +def take_image(detector): + """One small greyscale frame (PIL) of either kind of camera.""" + if is_observation_camera(detector): + # camera_name only names the folder ImSwitch saves the frame to. + response = client.post("/ExperimentController/snapOverviewImage", timeout=60, + params={"slot_id": "1", "camera_name": "hil_lightsource"}) + if response.status_code == 400: + raise LeftOut(f"observation camera not connected: {response.text}") + raw = base64.b64decode(client.json_of(response)["imageBase64"]) + frame = Image.open(io.BytesIO(raw)).convert("L") + return frame.resize((max(1, int(frame.width * FRAME_RESIZE)), + max(1, int(frame.height * FRAME_RESIZE)))) + response = client.get("/RecordingController/snapNumpyToFastAPI", timeout=60, + params={"detectorName": detector, "resizeFactor": FRAME_RESIZE}) + assert response.status_code == 200, response.text + return Image.open(io.BytesIO(response.content)).convert("L") + + +def image_difference(first, second): + """Mean absolute grey-value difference between two frames.""" + return ImageStat.Stat(ImageChops.difference(first, second)).mean[0] + + +def brightness(image): + """Mean grey value of a frame.""" + return ImageStat.Stat(image).mean[0] + + +def exposure_and_gain(detector): + """ "exposure 20, gain 0" as the camera reports them; None for a value it does not report.""" + values = [client.get_json("/SettingsController/getDetectorParameter", + {"detectorName": detector, "parameterName": name}) + for name in ("exposure", "gain")] + return f"exposure {values[0]}, gain {values[1]}" + + +def dark_baseline(detector): + """Measure the camera noise with the scene unchanged. + + Returns (last frame, noise floor, change a light must exceed). + """ + previous = take_image(detector) + drifts = [] + for _ in range(NOISE_SAMPLES - 1): + time.sleep(NOISE_DELAY_S) + current = take_image(detector) + drifts.append(image_difference(previous, current)) + previous = current + noise_floor = max(drifts) + required = max(MIN_CHANGE, noise_floor * NOISE_FACTOR) + print(f"\n{detector} dark noise: drifts={[round(d, 2) for d in drifts]}, " + f"noise_floor={noise_floor:.2f}, required_change={required:.2f}") + return previous, noise_floor, required + + +@contextlib.contextmanager +def live_view(detector): + """Keep an acquisition detector streaming inside a ``with`` block. + + Only a live view that was started here is stopped again. + """ + # The HTTP status is 200 even when ImSwitch refuses, so only the "status" field tells. + status = client.post_json("/LiveViewController/startLiveView", {"detectorName": detector}, + timeout=60).get("status") + if status == "long_exposure": + raise LeftOut("exposure too long for live view") + assert status in ("success", "already_running"), ( + f"{detector}: startLiveView answered {status!r}") + time.sleep(0.4) + try: + yield + finally: + if status == "success": + client.get_json("/LiveViewController/stopLiveView", {"detectorName": detector}) + + +def auto_expose(detector): + """One auto-exposure pass on the scene as it is now; needs a running live view.""" + client.get_json("/SettingsController/setDetectorExposureOnce", + {"detectorName": detector, "resetDelayMs": AUTO_EXPOSURE_RESET_MS}) + time.sleep(AUTO_EXPOSURE_RESET_MS / 1000 + 0.1) + + +@contextlib.contextmanager +def ready(detector): + """The camera ready to take frames inside a ``with`` block. + + An acquisition detector streams, and its exposure is set once, in the dark, + so that the dark, the lit and the switched-off frames share it. + The observation camera is left alone: it has no live view for the auto exposure to act on. + """ + if is_observation_camera(detector): + yield + return + with live_view(detector): + auto_expose(detector) + yield + + +def sees_switching(detector, switch): + """Switch the light on and off in front of one camera; ``switch(on)`` switches it. + + Returns what the camera did not see, as lines; empty when it saw both. + """ + with ready(detector): + dark, noise_floor, required = dark_baseline(detector) + switch(True) + lit = take_image(detector) + switch(False) + off = take_image(detector) + settings = exposure_and_gain(detector) + attach(f"{detector}: dark", dark) + attach(f"{detector}: on", lit) + attach(f"{detector}: off again", off) + on_change, off_change = image_difference(dark, lit), image_difference(dark, off) + print(f"\n{detector}: on changed {on_change:.2f}, off is {off_change:.2f} from dark, " + f"noise floor {noise_floor:.2f}, required {required:.2f}; brightness dark " + f"{brightness(dark):.2f}, on {brightness(lit):.2f}; {settings}") + missed = [] + if on_change < required: + missed.append(f"{detector}: on changed the image only by {on_change:.2f}, " + f"need >= {required:.2f} (noise floor {noise_floor:.2f}; {settings})") + # Off is judged against on, not against the noise: + # the observation camera's own exposure drifts after a bright light. + if off_change >= on_change / 2: + missed.append(f"{detector}: off is still {off_change:.2f} from dark, " + f"need < {on_change / 2:.2f} (half of what on changed)") + return missed + + +# --- fixtures --------------------------------------------------------------------------------- + +@pytest.fixture +def dark(): + """Every light off before and after the test.""" + all_lights_off() + yield + all_lights_off() + + +# --- tests ------------------------------------------------------------------------------------ + +@client.parametrize_from_rig("name", light_sources, "the setup has no lasers/LEDs") +def test_lightsource_reports_active(name): + """ImSwitch's own state follows the commands (at value 1); this does not prove light.""" + try: + client.get_json("/LaserController/setLaserValue", {"laserName": name, "value": 1}) + client.get_json("/LaserController/setLaserActive", {"laserName": name, "active": True}) + active = client.get_json("/LaserController/getLaserActive", {"laserName": name}) + value = client.get_json("/LaserController/getLaserValue", {"laserName": name}) + assert active is True, f"{name}: did not become active" + assert (value or 0) > 0, f"{name}: value is not positive" + + client.get_json("/LaserController/setLaserActive", {"laserName": name, "active": False}) + active = client.get_json("/LaserController/getLaserActive", {"laserName": name}) + assert active is False, f"{name}: did not become inactive" + finally: + set_light(name, False) + + +@client.parametrize_from_rig("name", leds, "no LED among the light sources of the setup") +def test_every_camera_sees_the_led_switch_on_and_off(name, dark): + found = cameras() + if not found: + pytest.skip("the setup has no camera that gives real frames") + missed, measured = [], 0 + # Every camera is measured before the verdict, so the report has the photos of all of them. + for detector in found: + try: + missed += sees_switching(detector, lambda on: set_light(name, on)) + measured += 1 + except LeftOut as reason: + note(f"{detector} left out: {reason}") + all_lights_off() + assert not missed, f"{name}:\n" + "\n".join(missed) + if not measured: + pytest.skip("no camera of the setup could give frames") diff --git a/imswitch/imcontrol/_test/hil/test_motor_axis_move.py b/imswitch/imcontrol/_test/hil/test_motor_axis_move.py new file mode 100644 index 000000000..02772c687 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_motor_axis_move.py @@ -0,0 +1,44 @@ +"""Each motor axis moves 10 µm and back, and ImSwitch reports the position it moved to. + +MOVES REAL HARDWARE (10 µm). + +The position is the step count of the firmware, not a measurement. +So this only shows that ImSwitch, the board and its bookkeeping follow the command. +""" +import pytest + +import hil_client as client + +STEP_UM = 10 +MOVE_TIMEOUT = 60 # seconds a blocking 10 µm move may take + + +def axes(): + """Every (positioner, axis) pair of the setup, e.g. ("ESP32Stage", "X").""" + return [(name, axis) for name, positioner in client.setup()["positioners"].items() + if positioner["forPositioning"] for axis in positioner["axes"]] + + +def position(positioner, axis): + return client.get_json("/PositionerController/getPositionerPositions")[positioner][axis] + + +def move(positioner, axis, distance): + """Move one axis by ``distance`` µm; returns once it has arrived.""" + client.get_json("/PositionerController/movePositioner", timeout=MOVE_TIMEOUT, params={ + "positionerName": positioner, "axis": axis, "dist": distance, "isBlocking": True}) + + +@client.parametrize_from_rig("positioner, axis", axes, "the setup has no positioner axes") +def test_axis_moves_by_step(positioner, axis): + before = position(positioner, axis) + try: + move(positioner, axis, STEP_UM) + after = position(positioner, axis) + assert after == pytest.approx(before + STEP_UM, abs=0.1), ( + f"{positioner} {axis}: {before} -> {after}, expected {before + STEP_UM}") + finally: + move(positioner, axis, -STEP_UM) + back = position(positioner, axis) + assert back == pytest.approx(before, abs=0.1), ( + f"{positioner} {axis}: back at {back}, expected {before}") diff --git a/imswitch/imcontrol/_test/hil/test_stage_homing.py b/imswitch/imcontrol/_test/hil/test_stage_homing.py new file mode 100644 index 000000000..9d890d7c5 --- /dev/null +++ b/imswitch/imcontrol/_test/hil/test_stage_homing.py @@ -0,0 +1,71 @@ +"""The stage homes Z, X and Y as "Safe Global Homing" does, and every axis reports done. + +MOVES REAL HARDWARE: Z, X and Y drive to their end stops. + +This is the button "Start homing" under FRAME settings: the endpoint startFrameHoming. +It homes Z and lifts it clear of the bottom, then homes X and Y. +At the end X and Y go to the transport position, and Z goes back to its height from before. +That is where the light tests find the stage. +The turret (A) is not homed here; the FRAME calibration homes it. + +The test passes when the homing ends as "done" with Z, X and Y "done", as the button shows it. +The positions at the start and at the end are in the report. +""" +import pytest + +import hil_client as client +from hil_report import note + +HOME_ORDER = "ZXY" # the axes startFrameHoming homes, in its order +POLL_S = 0.5 # seconds between two status reads while the stage homes +START_TIMEOUT = 10 # seconds the homing thread may take to start +HOME_EXTRA_S = 120 # seconds the whole homing may take beyond the homeTimeouts of the setup +DEFAULT_HOME_TIMEOUT_MS = 20000 # as ESP32StageManager, when the setup sets no homeTimeout + + +def positioner_to_home(): + """(name, managerProperties) of the first positioner with homing on for Z, X or Y.""" + for name, positioner in client.setup()["positioners"].items(): + properties = positioner["managerProperties"] + if any(properties.get(f"home{axis}enabled") for axis in HOME_ORDER): + return name, properties + return None, None + + +def homing_state(positioner): + return client.get_json("/PositionerController/getFrameHomingState", + {"positionerName": positioner}) + + +def positions(positioner): + return client.get_json("/PositionerController/getPositionerPositions")[positioner] + + +def test_stage_homes_with_safe_global_homing(): + positioner, properties = positioner_to_home() + if positioner is None: + pytest.skip("no positioner has homing switched on for Z, X or Y") + if homing_state(positioner)["active"]: + pytest.skip("a homing is already running") + timeout = HOME_EXTRA_S + sum( + properties.get(f"homeTimeout{axis}", DEFAULT_HOME_TIMEOUT_MS) / 1000 for axis in HOME_ORDER) + + note(f"positions at the start: {positions(positioner)}") + started = client.get_json("/PositionerController/startFrameHoming", + {"positionerName": positioner, "isBlocking": False}) + assert started.get("success"), f"startFrameHoming refused: {started}" + # The homing starts in a thread; until then the state is still the one of the last homing. + assert client.wait_until(lambda: homing_state(positioner)["active"], START_TIMEOUT), ( + f"the homing did not start within {START_TIMEOUT}s") + ended = client.wait_until(lambda: not homing_state(positioner)["active"], timeout, + interval=POLL_S) + state = homing_state(positioner) + if not ended: + client.get_json("/PositionerController/cancelFrameHoming", {"positionerName": positioner}) + pytest.fail(f"the homing did not end within {timeout:.0f}s, in phase " + f"{state['phase']!r}; it was cancelled") + note(f"positions at the end: {positions(positioner)}") + + assert state["phase"] == "done", f"the homing ended as {state['phase']!r}: {state['message']}" + not_done = [axis for axis in HOME_ORDER if state["axes"].get(axis) != "done"] + assert not not_done, f"not homed: {', '.join(not_done)} ({state['axes']})" diff --git a/imswitch/imcontrol/_test/unit/test_hil_reporting.py b/imswitch/imcontrol/_test/unit/test_hil_reporting.py new file mode 100644 index 000000000..847c8fe00 --- /dev/null +++ b/imswitch/imcontrol/_test/unit/test_hil_reporting.py @@ -0,0 +1,288 @@ +"""Offline checks of HIL orchestration and printable reports; never contact a rig.""" +import importlib.util +import io +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from pypdf import PdfReader +from reportlab.pdfgen.canvas import Canvas + +HIL = Path(__file__).parents[1] / "hil" +sys.path.insert(0, str(HIL)) +import hil_report +import hil_pdf + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def calibration_module(): + hil_report.reset() + return load("hil_calibration_test", HIL / "test_frame_calibration.py") + + +def pdf_bytes(text="Calibration evidence"): + data = io.BytesIO() + canvas = Canvas(data) + canvas.drawString(40, 700, text) + canvas.save() + return data.getvalue() + + +def test_pdf_has_the_failure_the_note_and_the_appended_pdf(tmp_path): + results = {"test_camera.py::test_capture": { + "status": "failed", "duration": 2., "details": "Diagnostic line\n" * 140 + "E LAST ERROR LINE"}} + hil_pdf.write_pdf(tmp_path / "hil-report.pdf", "FRAME.json", 0, results, + {"test_camera.py": "Every camera returns a frame."}, {}, + {"test_camera.py": ["A NOTE OF THE TEST"]}, + [("Calibration", pdf_bytes())]) + reader = PdfReader(tmp_path / "hil-report.pdf") + text = "\n".join(page.extract_text() for page in reader.pages[:-1]) + assert "LAST ERROR LINE" in text and "Every camera returns a frame." in text + assert "A NOTE OF THE TEST" in text + assert "Diagnostic line" not in text # only what went wrong, not the whole traceback + assert "Calibration evidence" in reader.pages[-1].extract_text() + assert len(reader.outline) == 2 + assert [file.name for file in tmp_path.iterdir()] == ["hil-report.pdf"] + + +def test_what_a_test_checks_is_the_whole_first_paragraph_of_its_docstring(): + hil_report.reset() + module = SimpleNamespace(__doc__="Every axis moves 10 um\nand steps back.\n\nMOVES HARDWARE.") + hil_report.ReportPlugin("out", "FRAME.json").pytest_itemcollected( + SimpleNamespace(nodeid="test_motor.py::test_it", module=module)) + assert hil_report._descriptions["test_motor.py"] == "Every axis moves 10 um and steps back." + + +def test_report_without_appended_pdf_has_only_the_test_pages(tmp_path): + hil_report.reset() + hil_report.record_problem("startup", "server unavailable") + hil_report.write_reports(tmp_path, "unavailable") + reader = PdfReader(tmp_path / "hil-report.pdf") + assert "server unavailable" in "".join(page.extract_text() for page in reader.pages) + assert len(reader.outline) == 1 + assert sorted(file.name for file in tmp_path.iterdir()) == ["hil-report.pdf", "summary.md"] + + +def test_pdf_failure_still_writes_the_summary(tmp_path, monkeypatch): + hil_report.reset() + def fail(*args): + raise ValueError("broken PDF") + monkeypatch.setattr(hil_pdf, "write_pdf", fail) + hil_report.write_reports(tmp_path, "FRAME.json") + assert hil_report.report_error == "broken PDF" + assert not (tmp_path / "hil-report.pdf").exists() + assert "broken PDF" in (tmp_path / "summary.md").read_text() + assert "hil-report.pdf` in the artifact" not in (tmp_path / "summary.md").read_text() + + +def test_a_long_image_list_fits_on_the_pages(tmp_path, monkeypatch): + # Run 43896090621: the pallet text in one table cell was taller than a page (LayoutError). + images = [f"ghcr.io/openuc2/image-{number}:sha-{number:07d}" for number in range(200)] + monkeypatch.setenv("HIL_IMAGES", "\n".join(images)) + results = {"test_board_link.py::test_it": {"status": "passed", "duration": 1., "details": ""}} + hil_pdf.write_pdf(tmp_path / "hil-report.pdf", "FRAME.json", 0, results, {}, {}, {}, []) + text = "".join(page.extract_text() for page in PdfReader(tmp_path / "hil-report.pdf").pages) + assert images[0] in text and images[-1] in text + + +def test_a_long_output_fits_on_the_pages(tmp_path): + # Many failed cases of one file: their lines must not end up in one cell taller than a page. + results = {f"test_motor.py::test_it[{axis}{number}]": { + "status": "failed", "duration": 1., "details": "E moved by 0\n" * 12} + for axis in "XYZA" for number in range(10)} + hil_pdf.write_pdf(tmp_path / "hil-report.pdf", "FRAME.json", 0, results, {}, {}, {}, []) + assert len(PdfReader(tmp_path / "hil-report.pdf").pages) > 2 + + +def test_summary_has_one_row_per_test_file(): + hil_report.reset() + hil_report._results.update({ + "test_board_link.py::test_a": {"status": "passed", "duration": 1., "details": ""}, + "test_lightsource.py::test_a[LED]": {"status": "failed", "duration": 2., + "details": "E LED: not seen"}, + "test_lightsource.py::test_b[LED]": {"status": "passed", "duration": 3., "details": ""}}) + lines = hil_report.summary("FRAME.json").splitlines() + assert "| 1 | board_link | passed | 1 passed | 1 s |" in lines + assert "| 2 | lightsource | **failed** | 1 failed, 1 passed | 5 s |" in lines + assert any("LED: not seen" in line for line in lines) + + +def test_stop_timeout_is_an_error(calibration_module, monkeypatch): + m = calibration_module + monkeypatch.setattr(m, "status", lambda: {"state": "running"}) + sent = [] + monkeypatch.setattr(m.client, "post_json", lambda endpoint, **k: sent.append(endpoint)) + monkeypatch.setattr(m.client, "wait_until", lambda *a, **k: False) + with pytest.raises(AssertionError, match="thread did not stop"): + m.stop_and_wait() + assert sent == ["/FrameCalibrationController/stopFrameCalibration"] + + +def test_poll_returns_at_terminal_status(calibration_module, monkeypatch): + m = calibration_module + monkeypatch.setattr(m.client, "post_json", lambda *a, **k: {"success": True}) + monkeypatch.setattr(m, "status", lambda: {"state": "finished", "startedAt": "now", "steps": []}) + monkeypatch.setattr(m.time, "sleep", lambda _: None) + run = {} + assert m.run_calibration(run, lambda line: None)["state"] == "finished" + assert run == {"accepted": True, "startedAt": "now"} + + +def test_export_rejects_other_run(calibration_module, monkeypatch): + m = calibration_module + monkeypatch.setattr(m, "running", lambda: False) + monkeypatch.setattr(m, "status", lambda: {"startedAt": "older"}) + with pytest.raises(AssertionError, match="identity changed"): + m.collect_report({"startedAt": "ours"}) + + +def test_failed_calibration_pdf_is_collected(calibration_module, monkeypatch, tmp_path): + m = calibration_module + monkeypatch.setenv("HIL_REPORT_DIR", str(tmp_path)) + monkeypatch.setattr(m, "running", lambda: False) + monkeypatch.setattr(m, "status", lambda: {"startedAt": "ours", "state": "failed", "message": "no focus"}) + monkeypatch.setattr(m.client, "get", lambda *a, **k: SimpleNamespace(status_code=200, content=pdf_bytes())) + m.collect_report({"startedAt": "ours"}) + (title, data), = hil_report._appended + assert data.startswith(b"%PDF-") + (notes,) = hil_report._notes.values() + assert notes[0] == "The calibration ended as failed: no focus" + assert "failed: no focus" in hil_report.summary("FRAME.json") + # Also saved on its own, so it survives a hil-report.pdf that cannot be written. + assert (tmp_path / "frame_calibration.pdf").read_bytes() == data + + +def test_calibration_progress_is_one_line_per_start_and_end(calibration_module): + steps = [{"name": "homing", "title": "Home", "status": "ok", "detail": "homed Z", "seconds": 12.4, + "warnings": ["slow"]}, + {"name": "objective0", "title": "First objective", "status": "running", "detail": "", + "seconds": None, "warnings": []}, + {"name": "save", "title": "Save", "status": "pending", "detail": "", "seconds": None, + "warnings": []}] + lines, shown = [], set() + calibration_module.show_progress(steps, shown, lines.append) + calibration_module.show_progress(steps, shown, lines.append) # nothing changed: no new line + assert lines == ["calibration 1/3 Home ...", "calibration 1/3 ok (12 s) homed Z", + " warning: slow", "calibration 2/3 First objective ..."] + + +def test_tests_run_in_order_and_unlisted_ones_last(): + conf = load("hil_hooks", HIL / "conftest.py") + items = [SimpleNamespace(nodeid=f"test_{name}.py::test_it", fspath=HIL / f"test_{name}.py", + add_marker=lambda _: None) + for name in ("frame_calibration", "new_test", "motor_axis_move", "board_link")] + conf.pytest_collection_modifyitems(items) + assert [conf.name_of(item) for item in items] == [ + "board_link", "motor_axis_move", "frame_calibration", "new_test"] + + +def test_calibration_summary_overflow_keeps_last_step(tmp_path): + report = load("frame_pdf", HIL.parents[1] / "controller/controllers/frame_calibration_report.py") + status = {"state": "failed", "steps": [dict(title=f"Step {i}", status="failed", detail="long detail " * 100, + warnings=["warning " * 50]) for i in range(12)]} + path = tmp_path / "calibration.pdf" + report.write_pdf(path, status, {"target": "plate"}, []) + reader = PdfReader(path) + assert len(reader.pages) > 1 + assert "Step 11" in "".join(p.extract_text() for p in reader.pages) + + +def test_camera_preparation_failure_does_not_prevent_test_execution(tmp_path, monkeypatch): + main = load("hil_main", HIL / "__main__.py") + monkeypatch.setattr(sys, "argv", ["hil", str(tmp_path)]) + monkeypatch.setattr(main, "wait_for_imswitch", lambda: None) + monkeypatch.setattr(main, "prepare_cameras", lambda: {"camera A": "no frame"}) + monkeypatch.setattr(main.client, "setup_file", lambda: "FRAME.json") + called = [] + def tests(*args, **kwargs): + called.append(True) + return pytest.ExitCode.OK + monkeypatch.setattr(main.pytest, "main", tests) + assert main.main() == main.FAILED + assert called == [True] + assert hil_report.counts()["error"] == 1 + + +def test_unreachable_server_still_writes_report(tmp_path, monkeypatch): + main = load("hil_main", HIL / "__main__.py") + (tmp_path / "hil-report.pdf").write_bytes(pdf_bytes("OLD RUN")) + monkeypatch.setattr(sys, "argv", ["hil", str(tmp_path)]) + def unavailable(): + raise AssertionError("server unavailable") + monkeypatch.setattr(main, "wait_for_imswitch", unavailable) + assert main.main() == main.COULD_NOT_RUN + text = "".join(page.extract_text() for page in PdfReader(tmp_path / "hil-report.pdf").pages) + assert "server unavailable" in text and "OLD RUN" not in text + + +def test_failed_calibration_is_stopped_and_its_report_collected(calibration_module, monkeypatch): + m = calibration_module + called = [] + def fails(run, live): + run.update(accepted=True, startedAt="ours") + raise AssertionError("original failure") + monkeypatch.setattr(m, "run_calibration", fails) + monkeypatch.setattr(m, "stop_and_wait", lambda: called.append("stop")) + monkeypatch.setattr(m, "collect_report", lambda run: called.append("pdf")) + monkeypatch.setattr(m, "all_lights_off", lambda: called.append("off")) + + # As pytest does it: the fixtures start in the order of ``calibration``'s arguments, + # and end in the reverse order. + run_fixture = m.run.__wrapped__() + lights_fixture = m.lights_off_afterwards.__wrapped__() + run = next(run_fixture) + next(lights_fixture) + with pytest.raises(AssertionError, match="original failure"): + m.calibration.__wrapped__(lambda line: None, run, None, None) + for fixture in (lights_fixture, run_fixture): + with pytest.raises(StopIteration): + next(fixture) + assert called == ["stop", "off", "pdf"] + + +def test_a_refused_calibration_is_not_stopped_and_gives_no_report(calibration_module, monkeypatch): + m = calibration_module + def refused(run, live): + raise AssertionError("startFrameCalibration refused") + monkeypatch.setattr(m, "run_calibration", refused) + monkeypatch.setattr(m, "stop_and_wait", lambda: pytest.fail("stopped another run")) + monkeypatch.setattr(m, "collect_report", lambda run: pytest.fail("took another run's report")) + run_fixture = m.run.__wrapped__() + run = next(run_fixture) + with pytest.raises(AssertionError, match="refused"): + m.calibration.__wrapped__(lambda line: None, run, None, None) + with pytest.raises(StopIteration): + next(run_fixture) + + +@pytest.mark.parametrize("failing, watched, runs", [ + ("firmware_server", "frame_calibration", True), # another firmware version can still calibrate + ("board_link", "frame_calibration", False), + ("motor_axis_move", "frame_calibration", False), +]) +def test_a_test_does_not_start_after_a_failure_it_depends_on(tmp_path, failing, watched, runs): + import os + import subprocess + conf = (HIL / "conftest.py").read_text() + (tmp_path / "conftest.py").write_text(conf + "\nclient.reachable = lambda: True\n") + (tmp_path / f"test_{failing}.py").write_text("def test_it():\n assert False, 'offline simulated failure'\n") + (tmp_path / f"test_{watched}.py").write_text( + "from pathlib import Path\ndef test_it(live):\n live('LIVE LINE')\n" + " Path(__file__).with_suffix('.ran').touch()\n") + env = dict(os.environ, PYTHONPATH=str(HIL), PYTEST_DISABLE_PLUGIN_AUTOLOAD="1") + result = subprocess.run([sys.executable, "-m", "pytest", str(tmp_path), "--confcutdir", str(tmp_path), "-q", "-rs"], + env=env, capture_output=True, text=True, timeout=60) + assert result.returncode == 1, result.stdout + result.stderr + assert (tmp_path / f"test_{watched}.ran").exists() == runs + # live() prints although pytest captures the output of a passing test + assert ("LIVE LINE" in result.stdout) == runs + if not runs: + assert f"not started, because {failing} failed" in result.stdout diff --git a/imswitch/imcontrol/controller/controllers/frame_calibration_report.py b/imswitch/imcontrol/controller/controllers/frame_calibration_report.py index 6babb0f1e..661caf3d3 100644 --- a/imswitch/imcontrol/controller/controllers/frame_calibration_report.py +++ b/imswitch/imcontrol/controller/controllers/frame_calibration_report.py @@ -122,46 +122,83 @@ def wrap(s, n=110): with PdfPages(path) as pdf: fig = Figure(figsize=(8.27, 11.69)) y = 0.96 - fig.text(0.06, y, "FRAME calibration report", fontsize=16, weight="bold") - y -= 0.03 + + def lines(text, size=9, color="black", bold=False, indent=0.06, width=100): + nonlocal fig, y + for line in wrap(text, width).splitlines(): + if y < 0.07: + pdf.savefig(fig) + fig = Figure(figsize=(8.27, 11.69)) + y = 0.96 + fig.text(0.06, y, "FRAME calibration report (continued)", fontsize=14, weight="bold") + y -= 0.04 + fig.text(indent, y, line, fontsize=size, family="monospace", va="top", + color=color, weight="bold" if bold else "normal") + y -= max(0.016, size / 600) + y -= 0.007 + + lines("FRAME calibration report", size=16, bold=True) res = status.get("result") or {} cam = status.get("camera") or {} - for line in (f"state: {status.get('state')} started {status.get('startedAt')} finished {status.get('finishedAt')}", - wrap(status.get("message", ""), 100), f"report folder: {status.get('reportDir', '')}", - f"stored calibration: {res.get('timestamp', 'none')} camera: exposure {cam.get('exposureMs')} ms, " - f"gain {cam.get('gainDb')} dB ({cam.get('note', '')})"): - fig.text(0.06, y, line, fontsize=9, family="monospace", va="top") - y -= 0.022 * (1 + line.count("\n")) - y -= 0.01 + lines(f"state: {status.get('state')} started {status.get('startedAt')} finished {status.get('finishedAt')}") + lines(status.get("message", "")) + lines(f"report folder: {status.get('reportDir', '')}") + lines(f"stored calibration: {res.get('timestamp', 'none')} camera: exposure {cam.get('exposureMs')} ms, " + f"gain {cam.get('gainDb')} dB ({cam.get('note', '')})") for st in status.get("steps", []): head = f"[{st.get('status', ''):>7}] {st.get('title', '')}" + (f" ({st['seconds']} s)" if st.get("seconds") is not None else "") - body = "\n".join(filter(None, [wrap(st.get("detail", ""), 100)] + [wrap("warning: " + w, 100) for w in st.get("warnings", [])])) color = {"failed": "tab:red", "warn": "tab:orange", "ok": "tab:green"}.get(st.get("status"), "black") - fig.text(0.06, y, head, fontsize=9, family="monospace", va="top", color=color, weight="bold") - y -= 0.018 - if body: - fig.text(0.09, y, body, fontsize=7.5, family="monospace", va="top") - y -= 0.0145 * (1 + body.count("\n")) + 0.004 - if y < 0.12: - break + lines(head, color=color, bold=True) + lines(st.get("detail", ""), size=7.5, indent=0.09) + for warning in st.get("warnings", []): + lines("warning: " + warning, size=7.5, indent=0.09) if settings: - fig.text(0.06, 0.08, wrap("settings: " + ", ".join(f"{k}={v}" for k, v in settings.items()), 120), - fontsize=6.5, family="monospace", va="top", color="dimgray") + lines("settings: " + ", ".join(f"{k}={v}" for k, v in settings.items()), + size=6.5, color="dimgray", width=120) pdf.savefig(fig) - for i in range(0, len(figures), 2): + def needs_full_page(f): + return (len(wrap(f.get("title", ""), 95).splitlines()) > 1 or + len(wrap(f.get("expected", ""), 100).splitlines()) > 2 or + len(wrap(f.get("measured", ""), 100).splitlines()) > 3) + + i = 0 + while i < len(figures): fig = Figure(figsize=(8.27, 11.69)) - for j, f in enumerate(figures[i:i + 2]): + if needs_full_page(figures[i]): + f = figures[i] + y = 0.96 + lines(f"{f.get('step', '')}: {f.get('title', '')}", size=10, bold=True) img = cv2.imdecode(np.frombuffer(f["jpg"], np.uint8), cv2.IMREAD_COLOR) - top = 0.97 - j * 0.49 - ax = fig.add_axes([0.06, top - 0.33, 0.88, 0.31]) + if y < 0.5: + pdf.savefig(fig) + fig = Figure(figsize=(8.27, 11.69)) + y = 0.96 + ax = fig.add_axes([0.06, y - 0.34, 0.88, 0.32]) ax.imshow(img[..., ::-1]) ax.set_axis_off() - ok = {True: " - ok", False: " - FAILED"}.get(f.get("ok"), "") - fig.text(0.06, top, f"{f.get('step', '')}: {f.get('title', '')}{ok}", fontsize=10, weight="bold", va="top", - color="tab:red" if f.get("ok") is False else "black") - fig.text(0.06, top - 0.345, wrap("expected (yellow): " + f.get("expected", ""), 105), fontsize=8, va="top", color="#a07800") - fig.text(0.06, top - 0.395, wrap("measured (green): " + f.get("measured", ""), 105), fontsize=8, va="top", color="darkgreen") + y -= 0.36 + lines("expected (yellow): " + f.get("expected", ""), size=8, color="#a07800") + lines("measured (green): " + f.get("measured", ""), size=8, color="darkgreen") + if f.get("ok") is not None: + lines("Result: " + ("ok" if f["ok"] else "FAILED"), size=9) + i += 1 + else: + group = [figures[i]] + if i + 1 < len(figures) and not needs_full_page(figures[i + 1]): + group.append(figures[i + 1]) + for j, f in enumerate(group): + img = cv2.imdecode(np.frombuffer(f["jpg"], np.uint8), cv2.IMREAD_COLOR) + top = 0.97 - j * 0.49 + ax = fig.add_axes([0.06, top - 0.33, 0.88, 0.31]) + ax.imshow(img[..., ::-1]) + ax.set_axis_off() + ok = {True: " - ok", False: " - FAILED"}.get(f.get("ok"), "") + fig.text(0.06, top, f"{f.get('step', '')}: {f.get('title', '')}{ok}", fontsize=10, weight="bold", va="top", + color="tab:red" if f.get("ok") is False else "black") + fig.text(0.06, top - 0.345, wrap("expected (yellow): " + f.get("expected", ""), 105), fontsize=8, va="top", color="#a07800") + fig.text(0.06, top - 0.395, wrap("measured (green): " + f.get("measured", ""), 105), fontsize=8, va="top", color="darkgreen") + i += len(group) pdf.savefig(fig) diff --git a/pyproject.toml b/pyproject.toml index d7861cb92..22589199b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,8 @@ dependencies = [ "fastapi >= 0.86.0", "uvicorn[standard] >= 0.19.0", "matplotlib == 3.9.2", + "reportlab>=4,<5", + "pypdf>=6,<7", "aiortc >= 1.9.0", "jupyter", "python-multipart >= 0.0.5", diff --git a/uv.lock b/uv.lock index 37bd13ddd..0a553a44d 100644 --- a/uv.lock +++ b/uv.lock @@ -1825,12 +1825,14 @@ dependencies = [ { name = "psutil" }, { name = "psygnal" }, { name = "pydantic" }, + { name = "pypdf" }, { name = "pyserial" }, { name = "python-dateutil" }, { name = "python-multipart" }, { name = "python-socketio" }, { name = "pyvisa-py" }, { name = "rekuest" }, + { name = "reportlab" }, { name = "requests" }, { name = "scikit-image" }, { name = "send2trash" }, @@ -1917,6 +1919,7 @@ requires-dist = [ { name = "pydantic", specifier = "==2.12.5" }, { name = "pygame-ce", marker = "extra == 'slm'", specifier = ">=2.5" }, { name = "pymmcore-plus", marker = "extra == 'pymmcore'", specifier = ">=0.10" }, + { name = "pypdf", specifier = ">=6,<7" }, { name = "pyserial", specifier = ">=3.4" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "python-dateutil", specifier = ">=2.8.1" }, @@ -1924,6 +1927,7 @@ requires-dist = [ { name = "python-socketio", extras = ["asyncio"], specifier = ">=5.11,<6" }, { name = "pyvisa-py", specifier = "==0.4.1" }, { name = "rekuest", specifier = "==7.1.0" }, + { name = "reportlab", specifier = ">=4,<5" }, { name = "requests", specifier = ">=2.25" }, { name = "rpi-gpio", marker = "extra == 'lepmon'" }, { name = "scikit-image", specifier = "==0.25.2" }, @@ -4260,6 +4264,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl", hash = "sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e", size = 113890, upload-time = "2025-09-21T04:11:04.117Z" }, ] +[[package]] +name = "pypdf" +version = "6.21.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" }, +] + [[package]] name = "pyreadline3" version = "3.5.4" @@ -4593,6 +4606,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/eb/d1f7fbf532163df65fac1d27fafa58a709467370fbb73efa848c3a00c948/rekuest-7.1.0-py3-none-any.whl", hash = "sha256:df5616fdfe8e0d089d7433a6c0fe064573583a911b032c018a3618a9af588661", size = 140638, upload-time = "2026-09-29T22:42:03.105Z" }, ] +[[package]] +name = "reportlab" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/3f/b3861b7e40c9d66f4a04e018958d681d16b948bfd1963c962d43a8c23f66/reportlab-4.5.1.tar.gz", hash = "sha256:9fdf68f4de9171ec66acb4a5feed8f8ca2af43479e707a6fbb0daa75d88e5494", size = 3939748, upload-time = "2026-05-12T10:14:13.663Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/45/ea7fad10122440de6e845568d106bffdc456ca0e8a1d8ae10b46016087e4/reportlab-4.5.1-py3-none-any.whl", hash = "sha256:06fce8cb56c83307cfa4909cdf4e6a2ddbb44e5d6ef4d2edca896d7e9769f091", size = 1957812, upload-time = "2026-05-12T10:14:10.622Z" }, +] + [[package]] name = "requests" version = "2.32.5"