diff --git a/pyopia/__init__.py b/pyopia/__init__.py index 6ebac24f..326b86ff 100644 --- a/pyopia/__init__.py +++ b/pyopia/__init__.py @@ -1 +1 @@ -__version__ = "2.18.0" +__version__ = "2.19.0" diff --git a/pyopia/cli.py b/pyopia/cli.py index ce524c7c..c61995fb 100644 --- a/pyopia/cli.py +++ b/pyopia/cli.py @@ -530,23 +530,45 @@ def convert_raw_images(config_filename: str): def make_montage( stats_filename: pathlib.Path, output_filename: str = "montage.png", + scaled: bool = False, + rel_scale: float = 1.0, filter_variable: Annotated[Tuple[str, float, float], typer.Option()] = [ None, None, None, ], + msize: int = 1024, ): """Create a montage of particles + By default particles are subsampled and packed into a square canvas + (`pyopia.statistics.make_montage`). With `--scaled`, a circular montage is + created instead, packed largest first, where every particle is attempted (no + upfront subsampling) (`pyopia.statistics.make_montage_scaled`). + Parameters ---------- - config_filename : str - Config filename - output_filename: str + stats_filename : pathlib.Path + Path to a `-STATS.nc` file, as produced by `process` or `merge-mfdata` + output_filename : str Store montage figure to this filename + scaled : bool, optional + Create a scaled circular montage, by default False + rel_scale : float, optional + Only used with `--scaled`. Fraction (0-1) of the full canvas area used as the + circular placement boundary, by default 1.0. Set it proportional to relative + sample size when comparing several montages side by side, so packed density + stays a fair comparison rather than every montage looking equally full + regardless of how much data it represents. See + `pyopia.statistics.make_montage_scaled`'s own docstring for the full explanation. filter_variable: list Variables to filter on (name, min, max), e.g. ['depth', 5, None] + msize : int, optional + Size of the (square) montage canvas in pixels, by default 1024 """ + if rel_scale != 1.0 and not scaled: + raise typer.BadParameter("--rel-scale can only be used with --scaled") + print("[blue]LOAD STATS") xstats = pyopia.io.load_stats(str(stats_filename)) config = pyopia.io.steps_from_xstats(xstats) @@ -560,55 +582,23 @@ def make_montage( ) print("[blue]CREATING MONTAGE") - montage = pyopia.statistics.make_montage( - xstats.to_pandas(), - config["general"]["pixel_size"], - config["steps"]["statextract"]["export_outputpath"], - eyecandy=False, - ) - - print("[blue]STORING MONTAGE") - pyopia.plotting.montage_plot(montage, config["general"]["pixel_size"]) - plt.savefig(output_filename, dpi=300, bbox_inches="tight") - - -@app.command() -def make_montage_scaled( - stats_filename: pathlib.Path, - output_filename: str = "montage.png", - rel_scale: float = 1.0, -): - """Create a scaled circular montage of particles, packed largest first - - Unlike `make-montage`, every particle is attempted (no upfront subsampling), and - `rel_scale` controls how much of the canvas is available for packing - set it - proportional to relative sample size when comparing several montages side by side, - so packed density stays a fair comparison rather than every montage looking - equally full regardless of how much data it represents. See - `pyopia.statistics.make_montage_scaled`'s own docstring for the full explanation. - - Parameters - ---------- - stats_filename : pathlib.Path - Path to a `-STATS.nc` file, as produced by `process` or `merge-mfdata` - output_filename : str - Store montage figure to this filename - rel_scale : float, optional - Fraction (0-1) of the full canvas area used as the circular placement - boundary, by default 1.0 - """ - print("[blue]LOAD STATS") - xstats = pyopia.io.load_stats(str(stats_filename)) - config = pyopia.io.steps_from_xstats(xstats) - - print("[blue]CREATING MONTAGE") - montage = pyopia.statistics.make_montage_scaled( - xstats.to_pandas(), - config["general"]["pixel_size"], - config["steps"]["statextract"]["export_outputpath"], - rel_scale=rel_scale, - eyecandy=False, - ) + if scaled: + montage = pyopia.statistics.make_montage_scaled( + xstats.to_pandas(), + config["general"]["pixel_size"], + config["steps"]["statextract"]["export_outputpath"], + msize=msize, + rel_scale=rel_scale, + eyecandy=False, + ) + else: + montage = pyopia.statistics.make_montage( + xstats.to_pandas(), + config["general"]["pixel_size"], + config["steps"]["statextract"]["export_outputpath"], + msize=msize, + eyecandy=False, + ) print("[blue]STORING MONTAGE") pyopia.plotting.montage_plot(montage, config["general"]["pixel_size"]) diff --git a/pyopia/dataexport/ecotaxa.py b/pyopia/dataexport/ecotaxa.py index 0b1d5471..b83950fb 100644 --- a/pyopia/dataexport/ecotaxa.py +++ b/pyopia/dataexport/ecotaxa.py @@ -143,10 +143,13 @@ def create_bundle( else roi_dir ) - # Create statistics DataFrame, add classification best guess info for all particles - self.df_stats_export = pyopia.statistics.add_best_guesses_to_stats( - xstats.to_pandas() - ) + # Create statistics DataFrame, adding classification best guess info (only + # used for the label folder names) when label folders are requested + self.df_stats_export = xstats.to_pandas() + if make_label_folders: + self.df_stats_export = pyopia.statistics.add_best_guesses_to_stats( + self.df_stats_export + ) buffer = io.BytesIO() with zipfile.ZipFile(self.export_filename, mode="w") as zip: diff --git a/pyopia/instrument/holo.py b/pyopia/instrument/holo.py index dbfc9b04..9de501d8 100644 --- a/pyopia/instrument/holo.py +++ b/pyopia/instrument/holo.py @@ -147,9 +147,9 @@ class Reconstruct(): stack_clean : float defines amount of cleaning of stack (fraction of max value below which to zero) forward_filter_option : int - switch to control filtering in frequency domain (0=none,1=DC only,2=zero ferquency/default) + switch to control filtering in frequency domain (0=none (default), 1=DC only, 2=zero-frequency row and column) inverse_output_option : int - switch to control optional scaling of output intensity (0=square/default,1=linear) + switch to control output intensity (0=squared real part (default), 1=real part) Returns ------- @@ -192,7 +192,7 @@ def forward_transform(im, forward_filter_option=2): im : np.array hologram (usually background-corrected) forward_filter_option : int - filtering in frequency domain (0=none/default,1=DC only,2=zero ferquency) + filtering in frequency domain (0=none (default), 1=DC only, 2=zero-frequency row and column) Returns ------- @@ -224,16 +224,21 @@ def create_kernel(im, pixel_size, wavelength, n, offset, minZ, maxZ, stepZ): Parameters ---------- - im : np.arry + im : np.array hologram pixel_size : float pixel_size in microns per pixel (i.e. usually 4.4 for lisst-holo type of resolution) wavelength : float laser wavelength in nm + n : float + refractive index of the sample medium + offset : float + air-equivalent distance in mm from the sensor to the front of the sample volume + (air path plus glass path divided by the glass refractive index) minZ : float - minimum reconstruction distance in mm + minimum reconstruction depth within the sample volume in mm maxZ : float - maximum reconstruction distance in mm + maximum reconstruction depth within the sample volume in mm stepZ : float step size in mm (i.e. resolution of reconstruction between minZ and maxZ) @@ -278,11 +283,11 @@ def inverse_transform(im_fft, kern, im_stack, inverse_output_option=0): im_stack : np.array pre-allocated array to receive output inverse_output_option: int - optional scaling of output intensity (0=square/default,1=linear) + output intensity (0=squared real part (default), 1=real part) Returns ------- - np.arry + np.array im_stack ''' @@ -304,7 +309,7 @@ def clean_stack(im_stack, stack_clean): ---------- im_stack : np.array - stack_clean : flaot + stack_clean : float pixels below this value will be zeroed Returns @@ -319,34 +324,34 @@ def clean_stack(im_stack, stack_clean): def std_map(im_stack): - '''_summary_ + '''summarise an image stack as the standard deviation through z Parameters ---------- - im_stack : _type_ - _description_ + im_stack : np.array + image stack Returns ------- - _type_ - _description_ + np.array + 2D image of the standard deviation through the stack ''' std_map = np.std(im_stack, axis=2) return std_map def max_map(im_stack): - '''_summary_ + '''summarise an image stack as the maximum through z Parameters ---------- - im_stack : _type_ - _description_ + im_stack : np.array + image stack Returns ------- - _type_ - _description_ + np.array + 2D image of the maximum through the stack ''' max_map = np.max(im_stack, axis=2) return max_map @@ -425,7 +430,8 @@ def find_focus_sobel(im_stack, bbox, increase_depth_of_field): Returns ------- im : image - focussed image for bbox + sobel edge magnitude of the bbox region at the focus plane + (or its maximum over adjacent planes if increase_depth_of_field is True) ifocus: int index through stack of focussed image @@ -543,7 +549,7 @@ def __call__(self, data): case _: raise ValueError('focus_function in pyopia.instrument.holo.Focus not recognised') - if self.discard_end_slices and (ifocus_ == 0 or ifocus_ == im_stack.shape[2]): + if self.discard_end_slices and (ifocus_ == 0 or ifocus_ == im_stack.shape[2] - 1): continue ifocus.append(ifocus_) rp_out.append(rp) @@ -600,7 +606,7 @@ def __call__(self, data): z = (np.arange(data['settings']['steps']['initial']['minZ'], (data['settings']['steps']['initial']['maxZ'] + data['settings']['steps']['initial']['stepZ']), data['settings']['steps']['initial']['stepZ'])) - stats['z'] = z[stats['ifocus']-1] + stats['z'] = z[stats['ifocus']] stats['holo_filename'] = data['filename'] data['stats'] = stats return data diff --git a/pyopia/process.py b/pyopia/process.py index db0695d8..3497d5b5 100644 --- a/pyopia/process.py +++ b/pyopia/process.py @@ -633,7 +633,7 @@ class Segment(): runs ndi.binary_fill_holes if True. Defaults to True. segment_source: (str, optional) The key in Pipeline.data of the image to be segmented. - Defaults to 'im_corrected' + Defaults to 'im_corrected' (suits silcam); holo pipelines should use 'im_focussed'. segmentation_method: (str, optional) Segmentation method to use: ``'fast'`` or ``'accurate'``. Defaults to ``'fast'``. diff --git a/pyopia/statistics.py b/pyopia/statistics.py index dd7d586e..e78a50fd 100644 --- a/pyopia/statistics.py +++ b/pyopia/statistics.py @@ -1221,6 +1221,11 @@ def add_best_guesses_to_stats(stats): ------- stats : DataFrame particle statistics from silcam process with new columns for best guess and best guess value + + Raises + ------ + ValueError + if stats has no "probability" columns (i.e. no classifier was used) """ cols = stats.columns @@ -1228,6 +1233,9 @@ def add_best_guesses_to_stats(stats): for i, c in enumerate(cols): p[i] = str(c).startswith("probability") + if not p.any(): + raise ValueError("stats contains no 'probability' columns - best guesses need classifier output") + pinds = np.squeeze(np.argwhere(p)) parray = np.array(stats.iloc[:, pinds[:]]) diff --git a/pyopia/tests/test_cli.py b/pyopia/tests/test_cli.py index b3c3581c..aa385e51 100644 --- a/pyopia/tests/test_cli.py +++ b/pyopia/tests/test_cli.py @@ -320,11 +320,11 @@ def test_make_montage_creates_real_montage_image(silcam_cli_merged_stats, tmp_pa @pytest.mark.slow -def test_make_montage_scaled_creates_real_montage_image(silcam_cli_merged_stats, tmp_path): +def test_make_montage_scaled_option_creates_real_montage_image(silcam_cli_merged_stats, tmp_path): montage_path = tmp_path / 'montage_scaled.png' result = invoke_in(tmp_path, [ - 'make-montage-scaled', str(silcam_cli_merged_stats), '--output-filename', str(montage_path) + 'make-montage', str(silcam_cli_merged_stats), '--scaled', '--output-filename', str(montage_path) ]) assert result.exit_code == 0, result.output @@ -349,6 +349,62 @@ def test_export_to_ecotaxa_creates_bundle_zip(silcam_cli_merged_stats, tmp_path) assert sum(name.endswith('.png') for name in names) == 1740 +@pytest.fixture +def silcam_stats_without_classifier(silcam_cli_merged_stats, tmp_path): + '''Copy of the merged real stats with the classifier `probability_*` columns removed''' + xstats = pyopia.io.load_stats(str(silcam_cli_merged_stats)) + xstats = xstats.drop_vars([v for v in xstats.data_vars if str(v).startswith('probability')]) + path = tmp_path / 'no-classifier-STATS.nc' + xstats.to_netcdf(path, engine=pyopia.io.NETCDF_ENGINE) + return path + + +@pytest.mark.slow +def test_export_to_ecotaxa_works_without_classifier_output(silcam_stats_without_classifier, tmp_path): + export_path = tmp_path / 'ecotaxa_export.zip' + + result = invoke_in(tmp_path, [ + 'export-to-ecotaxa', str(silcam_stats_without_classifier), str(export_path) + ]) + + assert result.exit_code == 0, result.output + with zipfile.ZipFile(export_path) as bundle: + names = bundle.namelist() + assert 'ecotaxa_particle_statistics.tsv' in names + assert sum(name.endswith('.png') for name in names) == 1740 + + +@pytest.mark.slow +def test_export_to_ecotaxa_label_folders_need_classifier_output(silcam_stats_without_classifier, tmp_path): + result = invoke_in(tmp_path, [ + 'export-to-ecotaxa', str(silcam_stats_without_classifier), str(tmp_path / 'ecotaxa_export.zip'), + '--make-label-folders' + ]) + + assert isinstance(result.exception, ValueError) + assert 'probability' in str(result.exception) + + +@pytest.mark.slow +def test_make_montage_accepts_msize_and_filter_variable(silcam_cli_merged_stats, tmp_path): + for extra_args in ([], ['--scaled']): + montage_path = tmp_path / f'montage{len(extra_args)}.png' + + result = invoke_in(tmp_path, [ + 'make-montage', str(silcam_cli_merged_stats), '--output-filename', str(montage_path), + '--msize', '512', '--filter-variable', 'major_axis_length', '0', '100000', *extra_args + ]) + + assert result.exit_code == 0, result.output + assert montage_path.stat().st_size > 0 + + +def test_make_montage_rel_scale_requires_scaled(tmp_path): + result = invoke_in(tmp_path, ['make-montage', 'unused-STATS.nc', '--rel-scale', '0.5']) + + assert result.exit_code == 2 + + @pytest.mark.slow def test_summary_stats_json_output(silcam_cli_merged_stats, tmp_path): result = invoke_in(tmp_path, ['summary-stats', str(silcam_cli_merged_stats), '--json-output']) diff --git a/pyopia/tests/test_holo.py b/pyopia/tests/test_holo.py new file mode 100644 index 00000000..03e6d4c5 --- /dev/null +++ b/pyopia/tests/test_holo.py @@ -0,0 +1,43 @@ +''' +Tests for pyopia.instrument.holo focus-depth bookkeeping, using small synthetic stacks. +''' + +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +import pyopia.instrument.holo as holo + + +def _stack_with_particle_at_plane(plane, n_planes=11): + im_stack = np.zeros((40, 40, n_planes)) + im_stack[:, :, :] = 0.1 + im_stack[16:24, 16:24, plane] = 1.0 + return im_stack + + +def _focus(im_stack): + focus = holo.Focus(stacksummary_function='max_map', threshold=0.97, focus_function='find_focus_imax') + return focus({'im_stack': im_stack}) + + +def test_focus_discards_particles_at_either_end_of_the_stack(): + assert _focus(_stack_with_particle_at_plane(5))['stack_ifocus'] == [5] + assert _focus(_stack_with_particle_at_plane(0))['stack_ifocus'] == [] + assert _focus(_stack_with_particle_at_plane(10))['stack_ifocus'] == [] + + +def test_mergestats_reports_the_depth_of_the_focus_plane(): + stats = pd.DataFrame({'minr': [16], 'minc': [16], 'maxr': [24], 'maxc': [24]}) + data = { + 'stats': stats, + 'stack_rp': [SimpleNamespace(bbox=(16, 16, 24, 24))], + 'stack_ifocus': [4], + 'filename': 'holo.pgm', + 'settings': {'steps': {'initial': {'minZ': 0, 'maxZ': 5, 'stepZ': 0.5}}}, + } + + z = holo.MergeStats()(data)['stats']['z'].iloc[0] + + assert z == 2.0 # plane index 4 of 0, 0.5, 1.0, ... diff --git a/pyopia/tests/test_pipeline.py b/pyopia/tests/test_pipeline.py index 432de882..247a4fc2 100644 --- a/pyopia/tests/test_pipeline.py +++ b/pyopia/tests/test_pipeline.py @@ -110,8 +110,8 @@ def test_holo_pipeline(tmp_path, holo_example_files): print('stats header: ', stats.data_vars) print('Total number of particles: ', len(stats.major_axis_length)) - assert len(stats.major_axis_length) == 40, ('Number of particles expected in this test is 56 for main' + - ' (or 40 for dev-1.2.)' + + assert len(stats.major_axis_length) == 39, ('Number of particles expected in this test is 39' + + ' (40 before particles focussing on the last slice were discarded)' + ' This test counted ' + str(len(stats.major_axis_length)) + ' Something has altered the number of particles detected')