From 52c18eb6489e5f1f2a56855db0d0a1ff34202fea Mon Sep 17 00:00:00 2001 From: Alex Nimmo Smith Date: Thu, 15 Oct 2026 21:50:31 +0100 Subject: [PATCH 1/4] Add a hologram simulator for testing reconstruction settings pyopia.simulator.holo simulates in-line holograms of opaque disks at known position, depth and size with the band-limited angular spectrum method. It can write a folder of holograms with their true particles, and match processed statistics back to them. Adds tests and a docs entry. --- docs/api.rst | 5 + pyopia/simulator/holo.py | 241 ++++++++++++++++++++++++++++ pyopia/tests/test_holo_simulator.py | 60 +++++++ 3 files changed, 306 insertions(+) create mode 100644 pyopia/simulator/holo.py create mode 100644 pyopia/tests/test_holo_simulator.py diff --git a/docs/api.rst b/docs/api.rst index 0539c670..702e1843 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -112,3 +112,8 @@ Silcam-Simulator -------------- .. automodule:: pyopia.simulator.silcam :members: + +Holo-Simulator +-------------- +.. automodule:: pyopia.simulator.holo + :members: diff --git a/pyopia/simulator/holo.py b/pyopia/simulator/holo.py new file mode 100644 index 00000000..ed2cad6b --- /dev/null +++ b/pyopia/simulator/holo.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +''' +Module containing tools for simulating in-line holograms of particles with known size and position +''' +import os + +import numpy as np +import pandas as pd +import scipy.fft as fft +import skimage.io + + +def _transfer_function(shape, pixel_size, wavelength, distance): + '''Band-limited angular spectrum transfer function for propagating a distance in a medium + + Parameters + ---------- + shape : tuple + (rows, columns) of the field + pixel_size : float + pixel size in metres + wavelength : float + wavelength in the propagation medium in metres + distance : float + propagation distance in metres + + Returns + ------- + np.array + complex transfer function, zero for evanescent and band-limited frequencies + ''' + ny, nx = shape + fx = fft.fftfreq(nx, pixel_size)[None, :] + fy = fft.fftfreq(ny, pixel_size)[:, None] + root = 1 - (wavelength * fx) ** 2 - (wavelength * fy) ** 2 + propagating = root > 0 + h = np.exp(2j * np.pi * distance / wavelength * np.sqrt(np.where(propagating, root, 0))) + + # band limit avoids aliasing of the transfer function at long distances + limit_x = 1 / (wavelength * np.sqrt((2 * distance / (nx * pixel_size)) ** 2 + 1)) + limit_y = 1 / (wavelength * np.sqrt((2 * distance / (ny * pixel_size)) ** 2 + 1)) + return h * propagating * (np.abs(fx) < limit_x) * (np.abs(fy) < limit_y) + + +def synthetic_hologram(shape, particles, pixel_size=4.4, wavelength=658, n=1.33, offset=27.0, + pad=512, mean_counts=160, noise_counts=1.0, rng=None): + '''Simulate an in-line hologram of opaque circular particles + + Each particle is a disk illuminated by a plane wave. Its scattered field (Babinet's principle) + is propagated to the sensor with the band-limited angular spectrum method [1]_, through the + medium (wavelength / n over the particle depth) and then through the offset (as air). The + fields of all particles are summed (single scattering) and the sensor records the intensity. + + Parameters + ---------- + shape : tuple + (rows, columns) of the hologram in pixels + particles : array + one row per particle: (x pixel, y pixel, depth in mm within the sample volume, diameter in microns) + pixel_size : float + pixel size in microns + wavelength : float + laser wavelength in nm + n : float + refractive index of the sample medium + offset : float + air-equivalent distance in mm from the sensor to the front of the sample volume + pad : int + pixels of zero padding on each side while propagating, to avoid wrap-around + mean_counts : float + mean grey level of the 8-bit hologram + noise_counts : float + standard deviation of added Gaussian noise in grey levels + rng : np.random.Generator, optional + random number generator for the noise + + Returns + ------- + np.array + 8-bit hologram + + References + ---------- + .. [1] Matsushima and Shimobaba, Optics Express 17, 19662 (2009) + ''' + rng = rng if rng is not None else np.random.default_rng() + ny, nx = shape + pitch = pixel_size * 1e-6 + wavelength_m = wavelength * 1e-9 + padded = (ny + 2 * pad, nx + 2 * pad) + + h_offset = _transfer_function(padded, pitch, wavelength_m, offset * 1e-3) + rows, cols = np.mgrid[0:padded[0], 0:padded[1]] + + scattered = np.zeros(padded, dtype=complex) + for x, y, depth, diameter in particles: + radius = diameter / pixel_size / 2 + x0, y0 = x + pad, y + pad + half = int(radius + 3) + window = (slice(int(y0) - half, int(y0) + half + 1), slice(int(x0) - half, int(x0) + half + 1)) + distance = np.hypot(cols[window] - x0, rows[window] - y0) + disk = np.zeros(padded) + disk[window] = np.clip(radius - distance + 0.5, 0, 1) # anti-aliased edge + + h_medium = _transfer_function(padded, pitch, wavelength_m / n, depth * 1e-3) + scattered += fft.ifft2(fft.fft2(disk) * h_offset * h_medium, workers=os.cpu_count()) + + field = (1 - scattered)[pad:pad + ny, pad:pad + nx] + image = np.abs(field) ** 2 * mean_counts + rng.normal(0, noise_counts, shape) + return np.clip(np.round(image), 0, 255).astype(np.uint8) + + +def random_particles(shape, count, rng, diameter_range=(20, 150), depth_range=(3, 47), + margin=80, min_separation=160): + '''Random particles for :func:`synthetic_hologram`, log-uniform in size and uniform in position and depth + + Parameters + ---------- + shape : tuple + (rows, columns) of the hologram in pixels + count : int + number of particles + rng : np.random.Generator + random number generator + diameter_range : tuple + minimum and maximum diameter in microns + depth_range : tuple + minimum and maximum depth in mm within the sample volume + margin : float + minimum distance of particle centres from the image edge in pixels + min_separation : float + minimum distance between particle centres in pixels + + Returns + ------- + np.array + one row per particle: (x pixel, y pixel, depth in mm, diameter in microns) + ''' + ny, nx = shape + particles = [] + while len(particles) < count: + x, y = rng.uniform(margin, nx - margin), rng.uniform(margin, ny - margin) + if all(np.hypot(x - px, y - py) > min_separation for px, py, *_ in particles): + diameter = np.exp(rng.uniform(np.log(diameter_range[0]), np.log(diameter_range[1]))) + particles.append((x, y, rng.uniform(*depth_range), diameter)) + return np.array(particles) + + +def write_synthetic_holograms(folder, n_images, particles_per_image, shape=(1200, 1600), seed=0, **kwargs): + '''Write simulated holograms to a folder, with the true particles in truth.csv + + Images are named like the example holograms (``001-2001.pgm``, ``001-2002.pgm``, ...). + + Parameters + ---------- + folder : str + output folder (created if needed) + n_images : int + number of holograms + particles_per_image : int + number of particles in each hologram + shape : tuple + (rows, columns) of each hologram in pixels + seed : int + seed for the particles and noise + **kwargs + passed to :func:`synthetic_hologram`, and to :func:`random_particles` for + diameter_range, depth_range, margin and min_separation + + Returns + ------- + truth : DataFrame + one row per particle with columns image, x_px (column), y_px (row), + z_mm (depth within the sample volume) and diameter_um; also saved as truth.csv + ''' + particle_args = {k: kwargs.pop(k) for k in ('diameter_range', 'depth_range', 'margin', 'min_separation') if k in kwargs} + rng = np.random.default_rng(seed) + os.makedirs(folder, exist_ok=True) + + rows = [] + for i in range(n_images): + name = f'001-{2001 + i}' + particles = random_particles(shape, particles_per_image, rng, **particle_args) + image = synthetic_hologram(shape, particles, rng=rng, **kwargs) + skimage.io.imsave(os.path.join(folder, name + '.pgm'), image, check_contrast=False) + rows += [(name, *particle) for particle in particles] + + truth = pd.DataFrame(rows, columns=['image', 'x_px', 'y_px', 'z_mm', 'diameter_um']) + truth.to_csv(os.path.join(folder, 'truth.csv'), index=False) + return truth + + +def match_to_truth(stats, truth, pixel_size=4.4): + '''Match processed particle statistics to the true particles of simulated holograms + + Each true particle is matched to the nearest unclaimed detection in the same image whose + centre lies within the larger of 8 pixels or the particle radius. + + Parameters + ---------- + stats : DataFrame + particle statistics from processing, with columns holo_filename, minr, minc, maxr, maxc, + equivalent_diameter (pixels) and z (mm) + truth : DataFrame + true particles, as returned by :func:`write_synthetic_holograms` + pixel_size : float + pixel size in microns + + Returns + ------- + matches : DataFrame + truth with added columns found, ecd_um (equivalent diameter), z_detected, + size_error (relative to the true diameter) and depth_error (mm); nan where not found + n_unmatched : int + number of detections not matched to a true particle + ''' + stats = stats.dropna(subset=['minr']).copy() + stats['image'] = [os.path.splitext(os.path.basename(f))[0] for f in stats['holo_filename']] + stats['x_px'] = (stats['minc'] + stats['maxc']) / 2 + stats['y_px'] = (stats['minr'] + stats['maxr']) / 2 + + matches = truth.copy() + for column in ('ecd_um', 'z_detected', 'size_error', 'depth_error'): + matches[column] = np.nan + matches['found'] = False + + claimed = set() + for i, particle in truth.iterrows(): + candidates = stats[(stats['image'] == particle['image']) & ~stats.index.isin(claimed)] + distance = np.hypot(candidates['x_px'] - particle['x_px'], candidates['y_px'] - particle['y_px']) + close = distance[distance < max(8, particle['diameter_um'] / pixel_size / 2)] + if close.empty: + continue + detection = candidates.loc[close.idxmin()] + claimed.add(close.idxmin()) + ecd = detection['equivalent_diameter'] * pixel_size + matches.loc[i, ['found', 'ecd_um', 'z_detected']] = [True, ecd, detection['z']] + matches.loc[i, 'size_error'] = (ecd - particle['diameter_um']) / particle['diameter_um'] + matches.loc[i, 'depth_error'] = detection['z'] - particle['z_mm'] + + return matches, len(stats) - len(claimed) diff --git a/pyopia/tests/test_holo_simulator.py b/pyopia/tests/test_holo_simulator.py new file mode 100644 index 00000000..28014176 --- /dev/null +++ b/pyopia/tests/test_holo_simulator.py @@ -0,0 +1,60 @@ +''' +Tests for pyopia.simulator.holo, using a small simulated hologram. +''' + +import numpy as np +import pandas as pd + +import pyopia.instrument.holo as holo +from pyopia.simulator.holo import match_to_truth, random_particles, synthetic_hologram, write_synthetic_holograms + + +def test_synthetic_hologram_is_reproducible_and_keeps_mean_level(): + particles = random_particles((200, 200), 3, np.random.default_rng(1), margin=40, min_separation=60) + + first = synthetic_hologram((200, 200), particles, pad=64, rng=np.random.default_rng(2)) + second = synthetic_hologram((200, 200), particles, pad=64, rng=np.random.default_rng(2)) + + assert first.dtype == np.uint8 + assert np.array_equal(first, second) + assert abs(first.mean() - 160) < 2 + + +def test_reconstruction_focuses_a_synthetic_particle_at_its_true_depth(): + true_depth = 20.0 + particles = np.array([[150.0, 150.0, true_depth, 100.0]]) + im = synthetic_hologram((300, 300), particles, pad=128, noise_counts=0) / 160.0 + + kern = holo.create_kernel(im, 4.4, 658, 1.33, 27, 0, 50, 0.5) + stack = holo.inverse_transform(holo.forward_transform(im, 2), kern, np.zeros(kern.shape)) + + focus = stack[140:160, 140:160, :].sum(axis=(0, 1)) + assert abs(np.argmax(focus) * 0.5 - true_depth) <= 1.0 + + +def test_write_synthetic_holograms_saves_images_and_matching_truth(tmp_path): + kwargs = dict(shape=(200, 200), pad=64, margin=40, min_separation=60) + + truth = write_synthetic_holograms(tmp_path, 2, 3, **kwargs) + + assert sorted(f.name for f in tmp_path.glob('*.pgm')) == ['001-2001.pgm', '001-2002.pgm'] + assert list(truth.columns) == ['image', 'x_px', 'y_px', 'z_mm', 'diameter_um'] + assert truth['image'].value_counts().to_dict() == {'001-2001': 3, '001-2002': 3} + pd.testing.assert_frame_equal(pd.read_csv(tmp_path / 'truth.csv'), truth) + assert write_synthetic_holograms(tmp_path / 'again', 2, 3, **kwargs).equals(truth) + + +def test_match_to_truth_reports_errors_missed_particles_and_extra_detections(): + truth = pd.DataFrame({'image': ['a', 'a'], 'x_px': [100.0, 300.0], 'y_px': [100.0, 300.0], + 'z_mm': [10.0, 20.0], 'diameter_um': [44.0, 88.0]}) + stats = pd.DataFrame({'holo_filename': ['/data/a.pgm', '/data/a.pgm'], + 'minr': [95, 500], 'minc': [95, 500], 'maxr': [105, 510], 'maxc': [105, 510], + 'equivalent_diameter': [11.0, 5.0], 'z': [10.5, 3.0]}) + + matches, n_unmatched = match_to_truth(stats, truth, pixel_size=4.4) + + assert matches['found'].tolist() == [True, False] + assert np.isclose(matches.loc[0, 'size_error'], (11.0 * 4.4 - 44.0) / 44.0) + assert matches.loc[0, 'depth_error'] == 0.5 + assert np.isnan(matches.loc[1, 'size_error']) + assert n_unmatched == 1 From 758e173a41f767a5b6dab7f52d4f730da8b882b2 Mon Sep 17 00:00:00 2001 From: Alex Nimmo Smith Date: Thu, 15 Oct 2026 21:50:31 +0100 Subject: [PATCH 2/4] Bump version to 2.20.0 --- pyopia/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyopia/__init__.py b/pyopia/__init__.py index 6ebac24f..3ea83e0a 100644 --- a/pyopia/__init__.py +++ b/pyopia/__init__.py @@ -1 +1 @@ -__version__ = "2.18.0" +__version__ = "2.20.0" From b31998e0448ebeceb40320a4c069f5b250b7974a Mon Sep 17 00:00:00 2001 From: Alex Nimmo Smith Date: Thu, 15 Oct 2026 22:13:11 +0100 Subject: [PATCH 3/4] Fix match_to_truth for stats with repeated index labels Processed stats restart their index for each image, so claiming detections by index label blocked matches in other images. Reset the index first, and add a test with repeated labels. --- pyopia/simulator/holo.py | 2 +- pyopia/tests/test_holo_simulator.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pyopia/simulator/holo.py b/pyopia/simulator/holo.py index ed2cad6b..457d0a95 100644 --- a/pyopia/simulator/holo.py +++ b/pyopia/simulator/holo.py @@ -214,7 +214,7 @@ def match_to_truth(stats, truth, pixel_size=4.4): n_unmatched : int number of detections not matched to a true particle ''' - stats = stats.dropna(subset=['minr']).copy() + stats = stats.dropna(subset=['minr']).reset_index(drop=True) # processed stats repeat index labels across images stats['image'] = [os.path.splitext(os.path.basename(f))[0] for f in stats['holo_filename']] stats['x_px'] = (stats['minc'] + stats['maxc']) / 2 stats['y_px'] = (stats['minr'] + stats['maxr']) / 2 diff --git a/pyopia/tests/test_holo_simulator.py b/pyopia/tests/test_holo_simulator.py index 28014176..46393d8e 100644 --- a/pyopia/tests/test_holo_simulator.py +++ b/pyopia/tests/test_holo_simulator.py @@ -58,3 +58,16 @@ def test_match_to_truth_reports_errors_missed_particles_and_extra_detections(): assert matches.loc[0, 'depth_error'] == 0.5 assert np.isnan(matches.loc[1, 'size_error']) assert n_unmatched == 1 + + +def test_match_to_truth_handles_stats_that_restart_their_index_for_each_image(): + truth = pd.DataFrame({'image': ['a', 'b'], 'x_px': [100.0, 100.0], 'y_px': [100.0, 100.0], + 'z_mm': [10.0, 10.0], 'diameter_um': [44.0, 44.0]}) + stats = pd.DataFrame({'holo_filename': ['a.pgm', 'b.pgm'], 'minr': [95, 95], 'minc': [95, 95], + 'maxr': [105, 105], 'maxc': [105, 105], 'equivalent_diameter': [10.0, 10.0], 'z': [10.0, 10.0]}, + index=[0, 0]) + + matches, n_unmatched = match_to_truth(stats, truth) + + assert matches['found'].tolist() == [True, True] + assert n_unmatched == 0 From cbde9bd0f2e53e65fa3825e7888203edf3488888 Mon Sep 17 00:00:00 2001 From: Alex Nimmo Smith Date: Fri, 16 Oct 2026 00:26:50 +0100 Subject: [PATCH 4/4] Fix the phase of the scattered field relative to the incident wave in synthetic_hologram The scattered field was propagated with the full carrier phase while the incident plane wave was not, so the interference term had an arbitrary phase that changed with distance. The transfer function is now relative to a normally incident plane wave. Adds a test that the shadow behind an opaque disk is dark at any distance, and corrects the expected mean level. --- pyopia/simulator/holo.py | 5 +++-- pyopia/tests/test_holo_simulator.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pyopia/simulator/holo.py b/pyopia/simulator/holo.py index 457d0a95..ab188f7f 100644 --- a/pyopia/simulator/holo.py +++ b/pyopia/simulator/holo.py @@ -27,14 +27,15 @@ def _transfer_function(shape, pixel_size, wavelength, distance): Returns ------- np.array - complex transfer function, zero for evanescent and band-limited frequencies + complex transfer function relative to a normally incident plane wave (so it is 1 at zero frequency), + zero for evanescent and band-limited frequencies ''' ny, nx = shape fx = fft.fftfreq(nx, pixel_size)[None, :] fy = fft.fftfreq(ny, pixel_size)[:, None] root = 1 - (wavelength * fx) ** 2 - (wavelength * fy) ** 2 propagating = root > 0 - h = np.exp(2j * np.pi * distance / wavelength * np.sqrt(np.where(propagating, root, 0))) + h = np.exp(2j * np.pi * distance / wavelength * (np.sqrt(np.where(propagating, root, 0)) - 1)) # band limit avoids aliasing of the transfer function at long distances limit_x = 1 / (wavelength * np.sqrt((2 * distance / (nx * pixel_size)) ** 2 + 1)) diff --git a/pyopia/tests/test_holo_simulator.py b/pyopia/tests/test_holo_simulator.py index 46393d8e..8bce4ee8 100644 --- a/pyopia/tests/test_holo_simulator.py +++ b/pyopia/tests/test_holo_simulator.py @@ -17,7 +17,7 @@ def test_synthetic_hologram_is_reproducible_and_keeps_mean_level(): assert first.dtype == np.uint8 assert np.array_equal(first, second) - assert abs(first.mean() - 160) < 2 + assert 0.9 * 160 < first.mean() < 160 + 1 # particles remove light, so the mean sits just below the background level def test_reconstruction_focuses_a_synthetic_particle_at_its_true_depth(): @@ -71,3 +71,16 @@ def test_match_to_truth_handles_stats_that_restart_their_index_for_each_image(): assert matches['found'].tolist() == [True, True] assert n_unmatched == 0 + + +def test_shadow_behind_an_opaque_particle_is_dark_whatever_the_distance(): + # the scattered field must add to the incident plane wave with the right phase, so a change in + # distance of a fraction of a wavelength must not change the shadow + particle_x, particle_y, diameter = 100.0, 100.0, 100.0 + rows, cols = np.mgrid[0:200, 0:200] + inside = np.hypot(rows - particle_y, cols - particle_x) < 0.7 * diameter / 4.4 / 2 + + for depth in (2.0, 2.00005, 2.0001, 2.00015): + particles = np.array([[particle_x, particle_y, depth, diameter]]) + image = synthetic_hologram((200, 200), particles, offset=0.0, pad=128, mean_counts=100, noise_counts=0) + assert image[inside].mean() / 100 < 0.35