Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyopia/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "2.18.0"
__version__ = "2.19.0"
94 changes: 42 additions & 52 deletions pyopia/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -530,23 +530,45 @@ def convert_raw_images(config_filename: str):
def make_montage(
stats_filename: pathlib.Path,
output_filename: str = "montage.png",
scaled: bool = False,
rel_scale: float = 1.0,
filter_variable: Annotated[Tuple[str, float, float], typer.Option()] = [
None,
None,
None,
],
msize: int = 1024,
):
"""Create a montage of particles

By default particles are subsampled and packed into a square canvas
(`pyopia.statistics.make_montage`). With `--scaled`, a circular montage is
created instead, packed largest first, where every particle is attempted (no
upfront subsampling) (`pyopia.statistics.make_montage_scaled`).

Parameters
----------
config_filename : str
Config filename
output_filename: str
stats_filename : pathlib.Path
Path to a `-STATS.nc` file, as produced by `process` or `merge-mfdata`
output_filename : str
Store montage figure to this filename
scaled : bool, optional
Create a scaled circular montage, by default False
rel_scale : float, optional
Only used with `--scaled`. Fraction (0-1) of the full canvas area used as the
circular placement boundary, by default 1.0. Set it proportional to relative
sample size when comparing several montages side by side, so packed density
stays a fair comparison rather than every montage looking equally full
regardless of how much data it represents. See
`pyopia.statistics.make_montage_scaled`'s own docstring for the full explanation.
filter_variable: list
Variables to filter on (name, min, max), e.g. ['depth', 5, None]
msize : int, optional
Size of the (square) montage canvas in pixels, by default 1024
"""
if rel_scale != 1.0 and not scaled:
raise typer.BadParameter("--rel-scale can only be used with --scaled")

print("[blue]LOAD STATS")
xstats = pyopia.io.load_stats(str(stats_filename))
config = pyopia.io.steps_from_xstats(xstats)
Expand All @@ -560,55 +582,23 @@ def make_montage(
)

print("[blue]CREATING MONTAGE")
montage = pyopia.statistics.make_montage(
xstats.to_pandas(),
config["general"]["pixel_size"],
config["steps"]["statextract"]["export_outputpath"],
eyecandy=False,
)

print("[blue]STORING MONTAGE")
pyopia.plotting.montage_plot(montage, config["general"]["pixel_size"])
plt.savefig(output_filename, dpi=300, bbox_inches="tight")


@app.command()
def make_montage_scaled(
stats_filename: pathlib.Path,
output_filename: str = "montage.png",
rel_scale: float = 1.0,
):
"""Create a scaled circular montage of particles, packed largest first

Unlike `make-montage`, every particle is attempted (no upfront subsampling), and
`rel_scale` controls how much of the canvas is available for packing - set it
proportional to relative sample size when comparing several montages side by side,
so packed density stays a fair comparison rather than every montage looking
equally full regardless of how much data it represents. See
`pyopia.statistics.make_montage_scaled`'s own docstring for the full explanation.

Parameters
----------
stats_filename : pathlib.Path
Path to a `-STATS.nc` file, as produced by `process` or `merge-mfdata`
output_filename : str
Store montage figure to this filename
rel_scale : float, optional
Fraction (0-1) of the full canvas area used as the circular placement
boundary, by default 1.0
"""
print("[blue]LOAD STATS")
xstats = pyopia.io.load_stats(str(stats_filename))
config = pyopia.io.steps_from_xstats(xstats)

print("[blue]CREATING MONTAGE")
montage = pyopia.statistics.make_montage_scaled(
xstats.to_pandas(),
config["general"]["pixel_size"],
config["steps"]["statextract"]["export_outputpath"],
rel_scale=rel_scale,
eyecandy=False,
)
if scaled:
montage = pyopia.statistics.make_montage_scaled(
xstats.to_pandas(),
config["general"]["pixel_size"],
config["steps"]["statextract"]["export_outputpath"],
msize=msize,
rel_scale=rel_scale,
eyecandy=False,
)
else:
montage = pyopia.statistics.make_montage(
xstats.to_pandas(),
config["general"]["pixel_size"],
config["steps"]["statextract"]["export_outputpath"],
msize=msize,
eyecandy=False,
)

print("[blue]STORING MONTAGE")
pyopia.plotting.montage_plot(montage, config["general"]["pixel_size"])
Expand Down
11 changes: 7 additions & 4 deletions pyopia/dataexport/ecotaxa.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,13 @@ def create_bundle(
else roi_dir
)

# Create statistics DataFrame, add classification best guess info for all particles
self.df_stats_export = pyopia.statistics.add_best_guesses_to_stats(
xstats.to_pandas()
)
# Create statistics DataFrame, adding classification best guess info (only
# used for the label folder names) when label folders are requested
self.df_stats_export = xstats.to_pandas()
if make_label_folders:
self.df_stats_export = pyopia.statistics.add_best_guesses_to_stats(
self.df_stats_export
)

buffer = io.BytesIO()
with zipfile.ZipFile(self.export_filename, mode="w") as zip:
Expand Down
50 changes: 28 additions & 22 deletions pyopia/instrument/holo.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,9 @@ class Reconstruct():
stack_clean : float
defines amount of cleaning of stack (fraction of max value below which to zero)
forward_filter_option : int
switch to control filtering in frequency domain (0=none,1=DC only,2=zero ferquency/default)
switch to control filtering in frequency domain (0=none (default), 1=DC only, 2=zero-frequency row and column)
inverse_output_option : int
switch to control optional scaling of output intensity (0=square/default,1=linear)
switch to control output intensity (0=squared real part (default), 1=real part)

Returns
-------
Expand Down Expand Up @@ -192,7 +192,7 @@ def forward_transform(im, forward_filter_option=2):
im : np.array
hologram (usually background-corrected)
forward_filter_option : int
filtering in frequency domain (0=none/default,1=DC only,2=zero ferquency)
filtering in frequency domain (0=none (default), 1=DC only, 2=zero-frequency row and column)

Returns
-------
Expand Down Expand Up @@ -224,16 +224,21 @@ def create_kernel(im, pixel_size, wavelength, n, offset, minZ, maxZ, stepZ):

Parameters
----------
im : np.arry
im : np.array
hologram
pixel_size : float
pixel_size in microns per pixel (i.e. usually 4.4 for lisst-holo type of resolution)
wavelength : float
laser wavelength in nm
n : float
refractive index of the sample medium
offset : float
air-equivalent distance in mm from the sensor to the front of the sample volume
(air path plus glass path divided by the glass refractive index)
minZ : float
minimum reconstruction distance in mm
minimum reconstruction depth within the sample volume in mm
maxZ : float
maximum reconstruction distance in mm
maximum reconstruction depth within the sample volume in mm
stepZ : float
step size in mm (i.e. resolution of reconstruction between minZ and maxZ)

Expand Down Expand Up @@ -278,11 +283,11 @@ def inverse_transform(im_fft, kern, im_stack, inverse_output_option=0):
im_stack : np.array
pre-allocated array to receive output
inverse_output_option: int
optional scaling of output intensity (0=square/default,1=linear)
output intensity (0=squared real part (default), 1=real part)

Returns
-------
np.arry
np.array
im_stack
'''

Expand All @@ -304,7 +309,7 @@ def clean_stack(im_stack, stack_clean):
----------
im_stack : np.array

stack_clean : flaot
stack_clean : float
pixels below this value will be zeroed

Returns
Expand All @@ -319,34 +324,34 @@ def clean_stack(im_stack, stack_clean):


def std_map(im_stack):
'''_summary_
'''summarise an image stack as the standard deviation through z

Parameters
----------
im_stack : _type_
_description_
im_stack : np.array
image stack

Returns
-------
_type_
_description_
np.array
2D image of the standard deviation through the stack
'''
std_map = np.std(im_stack, axis=2)
return std_map


def max_map(im_stack):
'''_summary_
'''summarise an image stack as the maximum through z

Parameters
----------
im_stack : _type_
_description_
im_stack : np.array
image stack

Returns
-------
_type_
_description_
np.array
2D image of the maximum through the stack
'''
max_map = np.max(im_stack, axis=2)
return max_map
Expand Down Expand Up @@ -425,7 +430,8 @@ def find_focus_sobel(im_stack, bbox, increase_depth_of_field):
Returns
-------
im : image
focussed image for bbox
sobel edge magnitude of the bbox region at the focus plane
(or its maximum over adjacent planes if increase_depth_of_field is True)

ifocus: int
index through stack of focussed image
Expand Down Expand Up @@ -543,7 +549,7 @@ def __call__(self, data):
case _:
raise ValueError('focus_function in pyopia.instrument.holo.Focus not recognised')

if self.discard_end_slices and (ifocus_ == 0 or ifocus_ == im_stack.shape[2]):
if self.discard_end_slices and (ifocus_ == 0 or ifocus_ == im_stack.shape[2] - 1):
continue
ifocus.append(ifocus_)
rp_out.append(rp)
Expand Down Expand Up @@ -600,7 +606,7 @@ def __call__(self, data):
z = (np.arange(data['settings']['steps']['initial']['minZ'],
(data['settings']['steps']['initial']['maxZ'] + data['settings']['steps']['initial']['stepZ']),
data['settings']['steps']['initial']['stepZ']))
stats['z'] = z[stats['ifocus']-1]
stats['z'] = z[stats['ifocus']]
stats['holo_filename'] = data['filename']
data['stats'] = stats
return data
Expand Down
2 changes: 1 addition & 1 deletion pyopia/process.py
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,7 @@ class Segment():
runs ndi.binary_fill_holes if True. Defaults to True.
segment_source: (str, optional)
The key in Pipeline.data of the image to be segmented.
Defaults to 'im_corrected'
Defaults to 'im_corrected' (suits silcam); holo pipelines should use 'im_focussed'.
segmentation_method: (str, optional)
Segmentation method to use: ``'fast'`` or ``'accurate'``.
Defaults to ``'fast'``.
Expand Down
8 changes: 8 additions & 0 deletions pyopia/statistics.py
Original file line number Diff line number Diff line change
Expand Up @@ -1221,13 +1221,21 @@ def add_best_guesses_to_stats(stats):
-------
stats : DataFrame
particle statistics from silcam process with new columns for best guess and best guess value

Raises
------
ValueError
if stats has no "probability" columns (i.e. no classifier was used)
"""
cols = stats.columns

p = np.zeros_like(cols) != 0
for i, c in enumerate(cols):
p[i] = str(c).startswith("probability")

if not p.any():
raise ValueError("stats contains no 'probability' columns - best guesses need classifier output")

pinds = np.squeeze(np.argwhere(p))

parray = np.array(stats.iloc[:, pinds[:]])
Expand Down
60 changes: 58 additions & 2 deletions pyopia/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,11 +320,11 @@ def test_make_montage_creates_real_montage_image(silcam_cli_merged_stats, tmp_pa


@pytest.mark.slow
def test_make_montage_scaled_creates_real_montage_image(silcam_cli_merged_stats, tmp_path):
def test_make_montage_scaled_option_creates_real_montage_image(silcam_cli_merged_stats, tmp_path):
montage_path = tmp_path / 'montage_scaled.png'

result = invoke_in(tmp_path, [
'make-montage-scaled', str(silcam_cli_merged_stats), '--output-filename', str(montage_path)
'make-montage', str(silcam_cli_merged_stats), '--scaled', '--output-filename', str(montage_path)
])

assert result.exit_code == 0, result.output
Expand All @@ -349,6 +349,62 @@ def test_export_to_ecotaxa_creates_bundle_zip(silcam_cli_merged_stats, tmp_path)
assert sum(name.endswith('.png') for name in names) == 1740


@pytest.fixture
def silcam_stats_without_classifier(silcam_cli_merged_stats, tmp_path):
'''Copy of the merged real stats with the classifier `probability_*` columns removed'''
xstats = pyopia.io.load_stats(str(silcam_cli_merged_stats))
xstats = xstats.drop_vars([v for v in xstats.data_vars if str(v).startswith('probability')])
path = tmp_path / 'no-classifier-STATS.nc'
xstats.to_netcdf(path, engine=pyopia.io.NETCDF_ENGINE)
return path


@pytest.mark.slow
def test_export_to_ecotaxa_works_without_classifier_output(silcam_stats_without_classifier, tmp_path):
export_path = tmp_path / 'ecotaxa_export.zip'

result = invoke_in(tmp_path, [
'export-to-ecotaxa', str(silcam_stats_without_classifier), str(export_path)
])

assert result.exit_code == 0, result.output
with zipfile.ZipFile(export_path) as bundle:
names = bundle.namelist()
assert 'ecotaxa_particle_statistics.tsv' in names
assert sum(name.endswith('.png') for name in names) == 1740


@pytest.mark.slow
def test_export_to_ecotaxa_label_folders_need_classifier_output(silcam_stats_without_classifier, tmp_path):
result = invoke_in(tmp_path, [
'export-to-ecotaxa', str(silcam_stats_without_classifier), str(tmp_path / 'ecotaxa_export.zip'),
'--make-label-folders'
])

assert isinstance(result.exception, ValueError)
assert 'probability' in str(result.exception)


@pytest.mark.slow
def test_make_montage_accepts_msize_and_filter_variable(silcam_cli_merged_stats, tmp_path):
for extra_args in ([], ['--scaled']):
montage_path = tmp_path / f'montage{len(extra_args)}.png'

result = invoke_in(tmp_path, [
'make-montage', str(silcam_cli_merged_stats), '--output-filename', str(montage_path),
'--msize', '512', '--filter-variable', 'major_axis_length', '0', '100000', *extra_args
])

assert result.exit_code == 0, result.output
assert montage_path.stat().st_size > 0


def test_make_montage_rel_scale_requires_scaled(tmp_path):
result = invoke_in(tmp_path, ['make-montage', 'unused-STATS.nc', '--rel-scale', '0.5'])

assert result.exit_code == 2


@pytest.mark.slow
def test_summary_stats_json_output(silcam_cli_merged_stats, tmp_path):
result = invoke_in(tmp_path, ['summary-stats', str(silcam_cli_merged_stats), '--json-output'])
Expand Down
Loading
Loading