diff --git a/.github/workflows/mysql-integration.yml b/.github/workflows/mysql-integration.yml
index e0045a1..a13de9c 100644
--- a/.github/workflows/mysql-integration.yml
+++ b/.github/workflows/mysql-integration.yml
@@ -40,7 +40,7 @@ jobs:
python-version: "3.12"
- name: Install dependencies
- run: uv pip install --system pytest "setuptools<81" "datajoint==0.14.7" -e .
+ run: uv pip install --system pytest "setuptools<81" -e .
- name: Run MySQL integration tests
env:
diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
index ae57507..6622b33 100644
--- a/.github/workflows/python-app.yml
+++ b/.github/workflows/python-app.yml
@@ -31,7 +31,7 @@ jobs:
- name: Install dependencies
run: |
uv pip install --system flake8 pytest
- uv pip install --system "setuptools<81" "datajoint==0.14.7" -e .
+ uv pip install --system "setuptools<81" -e .
- name: Lint with flake8
run: |
diff --git a/README.md b/README.md
index 285aaca..7d4a249 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,21 @@ e.g. based on the template djimaging/djconfig/djconf_template.json.
Fill out the missing values; if you don't know how, ask someone in your group.
> ❗ Never upload this personal config file to GitHub.
+DataJoint 2 requires MySQL 8 with `utf8mb4`/`utf8mb4_bin` and named stores.
+The template uses a reference-only DataJoint store rooted at
+`/gpfs01/euler/data` for externally managed acquisition, stimulus, and model
+files. DataJoint does not write `` values, although filesystem tools
+such as ROI-mask editors may still require write access to adjacent folders.
+Traces, snippets, response vectors, field/presentation channel averages, ROI masks,
+and small analysis arrays/objects use `` and are stored directly in the
+database. Large stimulus and population matrices, full spatiotemporal receptive
+fields, high-resolution channel averages, correlation maps, and model objects
+use `` or `` in the separate writable
+`processed` store. Paths inserted into `` fields must be
+relative to the common reference root.
+JSON does not support comments, so these store roles are documented here rather
+than inside the config template.
+
### Create a user folder
Inside the root folder djimaging (not in djimaging/djimaging)
@@ -70,10 +85,10 @@ calling schema.drop() and confirm by entering yes.
## Local testing
-Install the test environment with DataJoint 0.14.7:
+Install the test environment with DataJoint 2.3.x:
```bash
-uv pip install pytest "setuptools<81" "datajoint==0.14.7" -e .
+uv pip install pytest "setuptools<81" -e .
```
Run unit tests:
@@ -87,7 +102,8 @@ Run integration tests against a temporary MySQL 8.0.43 server:
```bash
docker run --rm --name djimaging-mysql-test \
-e MYSQL_ROOT_PASSWORD=datajoint -e MYSQL_ROOT_HOST=% \
- -p 3307:3306 -d mysql:8.0.43
+ -p 3307:3306 -d mysql:8.0.43 \
+ --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
until docker exec djimaging-mysql-test mysqladmin ping -h 127.0.0.1 -pdatajoint; do sleep 1; done
DJ_TEST_MYSQL=1 DJ_HOST=127.0.0.1 DJ_PORT=3307 \
diff --git a/djimaging/autorois/autoshift_utils.py b/djimaging/autorois/autoshift_utils.py
index 7e78317..22a7d72 100644
--- a/djimaging/autorois/autoshift_utils.py
+++ b/djimaging/autorois/autoshift_utils.py
@@ -139,15 +139,18 @@ def compute_corr_map_match_indexes(
# Iterate over all possible shifts in both x and y directions within the range [-ShiftMax, ShiftMax]
for i, (dx, dy) in enumerate(dx_dy_pairs, start=1):
# Crop the shifted_video and stack_corr images to the same size
- cropped = corr_map[max(0, -dx):min(ref_corr_map.shape[0], ref_corr_map.shape[0] - dx),
- max(0, -dy):min(ref_corr_map.shape[1], ref_corr_map.shape[1] - dy)]
+ cropped = corr_map[max(0, -dx):min(corr_map.shape[0], corr_map.shape[0] - dx),
+ max(0, -dy):min(corr_map.shape[1], corr_map.shape[1] - dy)]
ref_cropped = ref_corr_map[max(0, dx):min(ref_corr_map.shape[0], ref_corr_map.shape[0] + dx),
max(0, dy):min(ref_corr_map.shape[1], ref_corr_map.shape[1] + dy)]
# Calculate the correlation coefficient between the cropped images
if metric == 'corr':
- score = np.corrcoef(cropped.flatten(), ref_cropped.flatten())[0, 1]
+ a = cropped.ravel() - cropped.mean()
+ b = ref_cropped.ravel() - ref_cropped.mean()
+ denom = np.linalg.norm(a) * np.linalg.norm(b)
+ score = np.dot(a, b) / denom if denom > 0 else 0.0
elif metric == 'mse':
score = -np.mean((cropped - ref_cropped) ** 2)
else:
@@ -165,6 +168,7 @@ def compute_corr_map_match_indexes(
ax2.imshow(cropped)
plt.show()
+ plt.close(fig)
# Store the correlation coefficient in the cross-correlation matrix
match_indexes[dx + shift_max, dy + shift_max] = score
@@ -176,9 +180,9 @@ def compute_corr_map_match_indexes(
v_max = np.max(match_indexes)
v_max_row_loc, v_max_col_loc = np.unravel_index(np.argmax(match_indexes), match_indexes.shape)
- # Shift the ROI mask
- x_shift = v_max_row_loc - shift_max
- y_shift = v_max_col_loc - shift_max
+ # Recover the shift: dx stored at index dx+shift_max, so dx = index-shift_max
+ x_shift = shift_max - v_max_row_loc
+ y_shift = shift_max - v_max_col_loc
if verbose:
print("ROI mask shifted by", x_shift, "pixels in x and", y_shift, "pixels in y")
diff --git a/djimaging/autorois/cellpose_wrapper.py b/djimaging/autorois/cellpose_wrapper.py
index da0c727..9ac02e5 100644
--- a/djimaging/autorois/cellpose_wrapper.py
+++ b/djimaging/autorois/cellpose_wrapper.py
@@ -30,7 +30,7 @@ def __init__(self, init_kwargs: dict, eval_kwargs: dict) -> None:
from cellpose import models, io
io.logger_setup()
- self.model = models.Cellpose(**self.init_kwargs)
+ self.model = models.CellposeModel(**self.init_kwargs)
@staticmethod
def stack_to_image(stack: np.ndarray, n_artifact: int) -> np.ndarray:
@@ -139,7 +139,7 @@ def create_mask_from_data(
if multiple_stacks and 'stitch_threshold' not in eval_kwargs:
eval_kwargs['stitch_threshold'] = 1e-9 # Stitch everything together
- masks, flows, styles, diams = self.model.eval(imgs, do_3D=do_3D, **eval_kwargs)
+ masks, flows, *_ = self.model.eval(imgs, do_3D=do_3D, **eval_kwargs)
if plot:
self.plot_results(imgs, masks, flows)
diff --git a/djimaging/autorois/corr_roi_mask_utils.py b/djimaging/autorois/corr_roi_mask_utils.py
index 32de8a9..408d0cc 100644
--- a/djimaging/autorois/corr_roi_mask_utils.py
+++ b/djimaging/autorois/corr_roi_mask_utils.py
@@ -1,6 +1,5 @@
+import heapq
import warnings
-from collections import deque
-from typing import Callable
import numpy as np
from matplotlib import pyplot as plt
@@ -50,8 +49,9 @@ class CorrRoiMask:
If ``True``, use channel 0; otherwise use channel 1. Default is
``True``.
grow_use_corr_map : bool, optional
- If ``True``, use the precomputed correlation map during growing instead
- of computing pairwise correlations on the fly. Default is ``False``.
+ If ``True``, precompute a mean-centered stack and use vectorised
+ seed-to-pixel correlation during growing instead of computing
+ pairwise correlations on the fly. Default is ``False``.
grow_threshold : float or None, optional
Correlation threshold used during growing. If ``None``, the
line-threshold is used. Default is ``None``.
@@ -119,22 +119,28 @@ def create_mask_from_data(
if len(kwargs) > 0:
warnings.warn(f'ignoring kwargs: {kwargs}')
- cut_x = np.array(self.cut_x)
- cut_x[0] = np.maximum(cut_x[0], n_artifact)
+ cut_x = (max(self.cut_x[0], n_artifact), self.cut_x[1])
stack = ch0_stack if self.use_ch0_stack else ch1_stack
- n_pix_min = self.n_pix_min or 2
- n_pix_min = int(np.maximum(np.round(self.min_area_um2 / (pixel_size_um[0] * pixel_size_um[1])), n_pix_min))
+ if stack is None:
+ raise ValueError("ch1_stack must be provided when use_ch0_stack=False")
- n_pix_max = self.n_pix_max or 2
- n_pix_max = int(np.maximum(np.round(self.max_area_um2 / (pixel_size_um[0] * pixel_size_um[1])), n_pix_max))
+ if self.n_pix_min is not None:
+ n_pix_min = self.n_pix_min
+ else:
+ n_pix_min = max(2, int(np.round(self.min_area_um2 / (pixel_size_um[0] * pixel_size_um[1]))))
- # At most twice as long as wide
- n_pix_max_x = int(
- np.maximum(np.sqrt(2 * self.max_area_um2), 1)) if self.n_pix_max_x is None else self.n_pix_max_x
- n_pix_max_z = int(
- np.maximum(np.sqrt(2 * self.max_area_um2), 1)) if self.n_pix_max_z is None else self.n_pix_max_z
+ if self.n_pix_max is not None:
+ n_pix_max = self.n_pix_max
+ else:
+ n_pix_max = max(2, int(np.round(self.max_area_um2 / (pixel_size_um[0] * pixel_size_um[1]))))
+
+ # At most twice as long as wide, converted to pixels
+ n_pix_max_x = max(1, int(np.sqrt(2 * self.max_area_um2) / pixel_size_um[
+ 0])) if self.n_pix_max_x is None else self.n_pix_max_x
+ n_pix_max_z = max(1, int(np.sqrt(2 * self.max_area_um2) / pixel_size_um[
+ 1])) if self.n_pix_max_z is None else self.n_pix_max_z
print(f'Creating ROI mask for stack of shape={stack.shape}, '
f'with n_pix_min={n_pix_min}, n_pix_max={n_pix_max}, '
@@ -233,7 +239,7 @@ def corr_map_rois(
n_pix_max_x: int = 5,
n_pix_max_z: int = 5,
n_pix_max: int = 25,
- n_pix_min: float = 3.,
+ n_pix_min: int = 3,
only_local_thresh_pixels: bool = False,
corr_map: np.ndarray | None = None,
grow_use_corr_map: bool = False,
@@ -264,7 +270,7 @@ def corr_map_rois(
Maximum ROI extent in z (pixels). Default is 5.
n_pix_max : int, optional
Maximum total number of pixels per ROI. Default is 25.
- n_pix_min : float, optional
+ n_pix_min : int, optional
Minimum number of pixels for a ROI to be kept. Default is 3.
only_local_thresh_pixels : bool, optional
If ``True``, only pixels above the threshold can be added during
@@ -273,8 +279,10 @@ def corr_map_rois(
Pre-computed correlation map. Computed from ``stack`` if ``None``.
Default is ``None``.
grow_use_corr_map : bool, optional
- If ``True``, use ``corr_map`` values during growing. Default is
- ``False``.
+ If ``True``, precompute a mean-centered stack and per-pixel norms
+ once, then use a vectorised seed-to-all-pixel correlation map during
+ growing instead of computing per-neighbour correlations on the fly.
+ Default is ``False``.
line_threshold : np.ndarray or None, optional
Pre-computed per-z-line threshold. Computed if ``None``. Default is
``None``.
@@ -301,11 +309,11 @@ def corr_map_rois(
corr_map, cut_x=cut_x, cut_z=cut_z, q=line_threshold_q, line_threshold_min=line_threshold_min)
nx, nz, nt = stack.shape
- seed_ixs, seed_iys = np.argwhere(corr_map >= line_threshold).T
+ seed_ixs, seed_izs = np.argwhere(corr_map >= line_threshold).T
# Sort by neighborhood correlation
- sort_idxs = np.argsort(corr_map[seed_ixs, seed_iys])[::-1]
+ sort_idxs = np.argsort(corr_map[seed_ixs, seed_izs])[::-1]
seed_ixs = seed_ixs[sort_idxs]
- seed_iys = seed_iys[sort_idxs]
+ seed_izs = seed_izs[sort_idxs]
roi_mask = np.zeros((nx, nz), dtype=int)
@@ -320,23 +328,32 @@ def corr_map_rois(
p_threshold = grow_threshold if grow_threshold is not None else line_threshold
- for seed_ix, seed_iz in zip(seed_ixs, seed_iys):
+ # Precompute centered stack and norms for efficient seed-to-pixel correlation
+ if grow_use_corr_map:
+ stack_float = stack.astype(float)
+ stack_centered = stack_float - stack_float.mean(axis=2, keepdims=True)
+ stack_norms = np.sqrt((stack_centered ** 2).sum(axis=2))
+ else:
+ stack_centered = None
+ stack_norms = None
+
+ for seed_ix, seed_iz in zip(seed_ixs, seed_izs):
if roi_mask[seed_ix, seed_iz] != 0:
continue
roi_pixels = corr_map_grow_roi(
seed_ix, seed_iz, n_pix_max=n_pix_max, n_pix_max_x=n_pix_max_x, n_pix_max_z=n_pix_max_z,
p_threshold=p_threshold, stack=stack, ignore_map=ignore_map,
- corr_map=corr_map if grow_use_corr_map else None)
+ stack_centered=stack_centered, stack_norms=stack_norms)
if len(roi_pixels) == 0:
continue
- roi_ixs, roi_iys = np.array(roi_pixels).T
+ roi_ixs, roi_izs = np.array(roi_pixels).T
if roi_ixs.size >= n_pix_min:
- roi_mask[roi_ixs, roi_iys] = roi_id
- ignore_map[roi_ixs, roi_iys] = True
+ roi_mask[roi_ixs, roi_izs] = roi_id
+ ignore_map[roi_ixs, roi_izs] = True
roi_id += 1
if plot:
@@ -398,7 +415,8 @@ def corr_map_grow_roi(
p_threshold: np.ndarray | float,
stack: np.ndarray,
ignore_map: np.ndarray | None = None,
- corr_map: np.ndarray | None = None,
+ stack_centered: np.ndarray | None = None,
+ stack_norms: np.ndarray | None = None,
) -> list:
"""Grow a single ROI from a seed pixel by collecting correlated neighbours.
@@ -423,14 +441,20 @@ def corr_map_grow_roi(
Pixels below the threshold are not added to the ROI.
stack : np.ndarray
3-D imaging stack of shape (nx, nz, nt) used for pairwise correlation
- when ``corr_map`` is ``None``.
+ when ``stack_centered`` is ``None``.
ignore_map : np.ndarray or None, optional
Boolean 2-D mask of shape (nx, nz); ``True`` pixels are skipped.
Default is ``None``.
- corr_map : np.ndarray or None, optional
- Pre-computed 2-D correlation map. If provided, its values are used
- instead of computing pairwise correlations on the fly. Default is
- ``None``.
+ stack_centered : np.ndarray or None, optional
+ Pre-computed mean-centered stack of shape (nx, nz, nt). When provided
+ together with ``stack_norms``, a vectorised seed-to-all-pixel
+ correlation map is computed at the start of the call instead of
+ evaluating per-neighbour correlations on the fly.
+ Default is ``None``.
+ stack_norms : np.ndarray or None, optional
+ Pre-computed per-pixel L2 norms of the centered traces, shape
+ (nx, nz). Required when ``stack_centered`` is given.
+ Default is ``None``.
Returns
-------
@@ -439,14 +463,39 @@ def corr_map_grow_roi(
"""
nx, nz, nt = stack.shape
visited = np.zeros((nx, nz), dtype=bool)
+ in_heap = np.zeros((nx, nz), dtype=bool)
if ignore_map is not None:
visited[ignore_map] = True
+ in_heap[ignore_map] = True
p_threshold = np.asarray(p_threshold)
if p_threshold.size == 1:
p_threshold = np.full(nz, p_threshold)
- queue = deque([(seed_ix, seed_iz, 1.)])
+ # Compute seed-to-all-pixel correlation map (vectorised) or prepare
+ # for on-the-fly per-neighbour computation.
+ seed_centered = None
+
+ if stack_centered is not None and stack_norms is not None:
+ seed_vec = stack_centered[seed_ix, seed_iz, :]
+ seed_norm = float(stack_norms[seed_ix, seed_iz])
+ if seed_norm > 0:
+ dots = np.einsum('ijk,k->ij', stack_centered, seed_vec)
+ with np.errstate(divide='ignore', invalid='ignore'):
+ seed_corr_map = np.where(stack_norms > 0, dots / (stack_norms * seed_norm), 0.0)
+ else:
+ seed_corr_map = np.zeros((nx, nz))
+ else:
+ seed_corr_map = None
+ # Precompute normalised seed trace for per-neighbour correlation
+ seed_trace = stack[seed_ix, seed_iz, :].astype(float)
+ seed_centered = seed_trace - seed_trace.mean()
+ seed_norm = float(np.sqrt((seed_centered ** 2).sum()))
+
+ # Max-heap via negated correlation (heapq is a min-heap)
+ in_heap[seed_ix, seed_iz] = True
+ heap = [(-1.0, seed_ix, seed_iz)]
+
roi_pixels = []
min_x, max_x = seed_ix, seed_ix
min_z, max_z = seed_iz, seed_iz
@@ -454,9 +503,12 @@ def corr_map_grow_roi(
reached_x_max = False
reached_z_max = False
- while queue and len(roi_pixels) < n_pix_max:
- ix, iz, corr_i = queue.pop()
+ while heap and len(roi_pixels) < n_pix_max:
+ neg_corr_i, ix, iz = heapq.heappop(heap)
+ corr_i = -neg_corr_i
+
if visited[ix, iz]:
+ # Pixel was invalidated by a spatial-bound update after being pushed
continue
visited[ix, iz] = True
@@ -466,32 +518,41 @@ def corr_map_grow_roi(
# update range
if not reached_x_max:
- min_x = np.minimum(ix, min_x)
- max_x = np.maximum(ix, max_x)
+ min_x = min(ix, min_x)
+ max_x = max(ix, max_x)
if (max_x - min_x) >= n_pix_max_x - 1:
visited[:min_x, :] = True
visited[max_x + 1:, :] = True
+ in_heap[:min_x, :] = True
+ in_heap[max_x + 1:, :] = True
reached_x_max = True
if not reached_z_max:
- min_z = np.minimum(iz, min_z)
- max_z = np.maximum(iz, max_z)
+ min_z = min(iz, min_z)
+ max_z = max(iz, max_z)
if (max_z - min_z) >= n_pix_max_z - 1:
visited[:, :min_z] = True
visited[:, max_z + 1:] = True
+ in_heap[:, :min_z] = True
+ in_heap[:, max_z + 1:] = True
reached_z_max = True
for dx, dz in [(-1, 0), (1, 0), (0, -1), (0, 1)]:
- new_ix, new_iy = ix + dx, iz + dz
- if 0 <= new_ix < nx and 0 <= new_iy < nz and not visited[new_ix, new_iy]:
- if corr_map is not None:
- new_corr_i = corr_map[new_ix, new_iy]
+ new_ix, new_iz = ix + dx, iz + dz
+ if 0 <= new_ix < nx and 0 <= new_iz < nz and not in_heap[new_ix, new_iz]:
+ in_heap[new_ix, new_iz] = True
+ if seed_corr_map is not None:
+ new_corr_i = seed_corr_map[new_ix, new_iz]
else:
- new_corr_i = np.corrcoef(stack[seed_ix, seed_iz, :], stack[new_ix, new_iy, :])[0, 1]
- queue.append((new_ix, new_iy, new_corr_i))
-
- queue = deque(sorted(queue, key=lambda v: v[2]))
+ neigh_trace = stack[new_ix, new_iz, :].astype(float)
+ neigh_centered = neigh_trace - neigh_trace.mean()
+ neigh_norm = float(np.sqrt((neigh_centered ** 2).sum()))
+ if seed_norm > 0 and neigh_norm > 0:
+ new_corr_i = float(np.dot(seed_centered, neigh_centered) / (seed_norm * neigh_norm))
+ else:
+ new_corr_i = 0.0
+ heapq.heappush(heap, (-new_corr_i, new_ix, new_iz))
return roi_pixels
diff --git a/djimaging/autorois/roi_canvas.py b/djimaging/autorois/roi_canvas.py
index 7af16d6..f379c1b 100644
--- a/djimaging/autorois/roi_canvas.py
+++ b/djimaging/autorois/roi_canvas.py
@@ -1,6 +1,6 @@
import os.path
-import pickle
import warnings
+from collections import deque
from datetime import datetime
from typing import Optional, Dict
@@ -222,7 +222,20 @@ def add_to_current_delete_mask(self, mask):
def add_to_current_delete_mask_at_loc(self, mask, ix, iy):
"""Add a potentially smaller mask to the current delete mask at location ix, iy"""
w, h = mask.shape
- self.current_delete_mask[ix - w // 2:ix + w // 2 + 1, iy - h // 2:iy + h // 2 + 1] |= mask.astype(bool)
+ i_l = ix - w // 2
+ i_r = ix + w // 2 + 1
+ i_b = iy - h // 2
+ i_t = iy + h // 2 + 1
+
+ if i_l > 0 and i_r < self.nx and i_b > 0 and i_t < self.ny:
+ self.current_delete_mask[i_l:i_r, i_b:i_t] |= mask.astype(bool)
+ else:
+ rm_l = abs(min(0, i_l))
+ rm_r = abs(min(0, self.nx - i_r))
+ rm_b = abs(min(0, i_b))
+ rm_t = abs(min(0, self.ny - i_t))
+ self.current_delete_mask[i_l + rm_l:i_r - rm_r, i_b + rm_b:i_t - rm_t] |= (
+ mask[rm_l:mask.shape[0] - rm_r, rm_b:mask.shape[1] - rm_t].astype(bool))
def load_current_mask(self):
"""Load current mask from all masks"""
@@ -308,18 +321,23 @@ def compute_autoshift(self, shift_max=5, fun_progress=None):
def update_roi_mask_img(self):
"""Update ROI masks image"""
- self.roi_mask_img = np.zeros_like(self.roi_mask_img)
+ self.roi_mask_img[:] = 0
if self._selected_view == 'none':
return
- new_roi_mask_img = np.zeros_like(self.roi_mask_img)
+ unique_rois = np.unique(self.roi_masks)[1:] # skip background label 0
+ if unique_rois.size == 0:
+ return
- for roi in np.unique(self.roi_masks)[1:]:
- xs, ys = np.where(self.roi_masks == roi)
- new_roi_mask_img[xs, ys, :] = self._get_roi_rgba255(roi)
+ # Build a per-label RGBA lookup table; index 0 = background (stays 0)
+ max_label = int(self.roi_masks.max())
+ lut = np.zeros((max_label + 1, 4), dtype=int)
+ for roi in unique_rois:
+ lut[roi] = self._get_roi_rgba255(roi)
- self.roi_mask_img = new_roi_mask_img
+ # Single vectorized index: no per-label np.where needed
+ self.roi_mask_img[:] = lut[self.roi_masks]
def print_state(self):
print('bg'.ljust(15), self._selected_bg)
@@ -392,7 +410,7 @@ def __init__(
self.canvas_width = canvas_width
# Create output widget
- self.log_messages = []
+ self.log_messages = deque(maxlen=26)
self.log_counter = 0
self.log_widget = Output()
@@ -484,11 +502,11 @@ def log(self, message):
return
self.log_counter += 1
- self.log_messages = [f"{self.log_counter}: {message}"] + self.log_messages[:25]
+ self.log_messages.appendleft(f"{self.log_counter}: {message}")
with self.log_widget:
self.log_widget.clear_output(wait=True)
- for message in self.log_messages:
- print(message)
+ for msg in self.log_messages:
+ print(msg)
def create_widget_canvas(self):
widget = MultiCanvas(n_canvases=4, width=self.npx, height=self.npy)
@@ -685,7 +703,7 @@ def set_selected_stim(self, value):
self.log(f'Setting stim: {value}')
self.widget_sel_stim.value = value
- self._selected_stim_idx = np.argmax([stim_name == value for stim_name in self.pres_names])
+ self._selected_stim_idx = self.pres_names.index(value)
self.update_roi_masks_backup()
self.output_file = self.output_files[self._selected_stim_idx]
@@ -711,7 +729,7 @@ def update_shift_widget(self):
def create_widget_sel_autorois(self):
"""Create and return button"""
- options = [key for key in self.autorois_models]
+ options = list(self.autorois_models)
widget = Dropdown(options=options, value=options[0], description='AutoROIs:', disabled=False)
def change(value):
@@ -1103,8 +1121,9 @@ def prep_roi_mask_for_file(self):
self.exec_save_current_roi()
roi_mask = self.roi_masks.copy()
roi_shift = self.shifts[self._selected_stim_idx]
- roi_shift = np.array([-roi_shift[0], -roi_shift[1]])
- roi_mask = mask_utils.shift_array(img=roi_mask, shift=roi_shift, inplace=True, cval=0)
+ # Negate: shift_array moves content in the opposite direction to the acquisition shift
+ inv_roi_shift = (-roi_shift[0], -roi_shift[1])
+ roi_mask = mask_utils.shift_array(img=roi_mask, shift=inv_roi_shift, inplace=True, cval=0)
return roi_mask
def _prep_save_to_file(self):
@@ -1127,8 +1146,8 @@ def exec_save_to_file(self, botton=None):
roi_mask, output_file = self._prep_save_to_file()
- with open(self.output_file, 'wb') as f:
- pickle.dump(roi_mask, f)
+ mask_utils.save_roi_mask_file(self.output_file, roi_mask,
+ mask_utils._infer_roi_file_format(self.output_file))
success = os.path.isfile(self.output_file)
if success:
@@ -1143,7 +1162,6 @@ def create_widget_save_all_to_file(self):
def exec_save_all_to_file(self, button=None, raise_error=False):
self.log('Saving all to file')
- import time
self.update_progress(0)
roi_masks, output_files = [], []
@@ -1159,19 +1177,18 @@ def exec_save_all_to_file(self, button=None, raise_error=False):
raise e
self.widget_save_info.description = 'Error!'.ljust(20)
warnings.warn(f'Error saving {stim}: {e}')
- time.sleep(0.5)
-
- for roi_mask, output_file in zip(roi_masks, output_files):
- with open(output_file, 'wb') as f:
- pickle.dump(roi_mask, f)
self.update_progress(100 * i / len(self.pres_names))
- success = all([os.path.isfile(output_file) for output_file in output_files])
- if success:
- self.widget_save_info.description = 'Success!'.ljust(20)
- else:
- self.widget_save_info.description = 'Error!'.ljust(20)
+ for roi_mask, output_file in zip(roi_masks, output_files):
+ mask_utils.save_roi_mask_file(output_file, roi_mask,
+ mask_utils._infer_roi_file_format(output_file))
+
+ success = all(os.path.isfile(f) for f in output_files)
+ if success:
+ self.widget_save_info.description = 'Success!'.ljust(20)
+ else:
+ self.widget_save_info.description = 'Error!'.ljust(20)
def create_widget_save_info(self):
widget = HTML(value=f'{self.output_file}', placeholder='output_file', description=''.ljust(20))
@@ -1200,7 +1217,8 @@ def update_info(self):
self.update_diagnostics(update=False)
def _compute_traces(self):
- return self.get_current_stack()[self.get_current_selected_roi_mask()]
+ mask = self.get_current_selected_roi_mask()
+ return self.get_current_stack()[mask]
def _compute_cc(self, traces):
if len(traces) <= 1:
@@ -1265,7 +1283,6 @@ def _apply_tool(self, x, y, down):
self.add_to_current_mask_at_loc(self.brush_mask, ix, iy)
elif self._selected_tool == 'erase':
self.log(f'Erasing from mask at x={x}, y={y}, ix={ix}, iy={iy}')
- self.add_to_current_mask_at_loc(np.zeros_like(self.brush_mask), ix, iy)
self.add_to_current_delete_mask_at_loc(self.brush_mask, ix, iy)
elif self._selected_tool == 'cc':
mask = mask_utils.get_mask_by_cc(
diff --git a/djimaging/djconfig/README.md b/djimaging/djconfig/README.md
index 78b84a9..25a9303 100644
--- a/djimaging/djconfig/README.md
+++ b/djimaging/djconfig/README.md
@@ -1,3 +1,6 @@
-You need a working DJ configuratin to create a DataBase.
-This folder provides a default, with dummy values.
-Make sure to never upload a real configuration to GitHub because it may contain sensitive information.
\ No newline at end of file
+You need a working DataJoint configuration to create a database. This folder
+provides a DataJoint 2 template with dummy values and the standard
+`acquisition`, `processed`, and `models` stores.
+
+Keep store credentials in DataJoint's `.secrets` files or environment variables,
+never in the JSON configuration. Do not upload a real personal configuration.
diff --git a/djimaging/djconfig/djconf_template.json b/djimaging/djconfig/djconf_template.json
index f0247be..6f9318f 100644
--- a/djimaging/djconfig/djconf_template.json
+++ b/djimaging/djconfig/djconf_template.json
@@ -2,18 +2,27 @@
"database.host": "",
"database.password": "",
"database.user": "",
- "database.port": 0,
+ "database.port": 3306,
"database.reconnect": true,
"connection.init_function": null,
- "connection.charset": "",
+ "connection.charset": "utf8mb4",
"loglevel": "INFO",
"safemode": true,
- "fetch_format": "array",
+ "download_path": "./.datajoint-downloads",
"display.limit": 20,
"display.width": 14,
"display.show_tuple_count": true,
"database.use_tls": null,
- "enable_python_native_blobs": true,
- "stores": null,
- "schema_name": ""
-}
\ No newline at end of file
+ "stores": {
+ "default": "processed",
+ "filepath_default": "reference",
+ "reference": {
+ "protocol": "file",
+ "location": "/gpfs01/euler/data"
+ },
+ "processed": {
+ "protocol": "file",
+ "location": "/path/to/processed"
+ }
+ }
+}
diff --git a/djimaging/notebooks/connect_to_schema.ipynb b/djimaging/notebooks/connect_to_schema.ipynb
index 97b8263..976ddd1 100644
--- a/djimaging/notebooks/connect_to_schema.ipynb
+++ b/djimaging/notebooks/connect_to_schema.ipynb
@@ -17,8 +17,10 @@
"id": "4bc2de48-c11a-407f-aa22-f6b3ff06f37d",
"metadata": {},
"source": [
- "import datajoint as dj\n",
- "import os"
+ "import os\n",
+ "os.environ[\"DJ_IGNORE_CONFIG_FILE\"] = \"true\"\n",
+ "\n",
+ "import datajoint as dj"
],
"outputs": [],
"execution_count": null
@@ -99,7 +101,7 @@
"cell_type": "code",
"id": "876f6549-124b-411e-bdd0-a447cbece053",
"metadata": {},
- "source": "schema.log.fetch(format='frame').iloc[-1]",
+ "source": "schema.log.to_pandas().iloc[-1]",
"outputs": [],
"execution_count": null
},
@@ -109,7 +111,7 @@
"metadata": {},
"source": [
"# Can first and last log timestamp\n",
- "timestamps, users = schema.log.fetch(\"timestamp\", \"user\")\n",
+ "timestamps, users = schema.log.to_arrays(\"timestamp\", \"user\")\n",
"timestamps[0], timestamps[-1]"
],
"outputs": [],
diff --git a/djimaging/notebooks/scan_for_corrupted_recordings.ipynb b/djimaging/notebooks/scan_for_corrupted_recordings.ipynb
index 3c480d8..f64d798 100644
--- a/djimaging/notebooks/scan_for_corrupted_recordings.ipynb
+++ b/djimaging/notebooks/scan_for_corrupted_recordings.ipynb
@@ -14,8 +14,10 @@
"metadata": {},
"cell_type": "code",
"source": [
- "import datajoint as dj\n",
- "import os"
+ "import os\n",
+ "os.environ[\"DJ_IGNORE_CONFIG_FILE\"] = \"true\"\n",
+ "\n",
+ "import datajoint as dj"
],
"id": "c4a531aca82df872",
"outputs": [],
@@ -88,7 +90,7 @@
{
"metadata": {},
"cell_type": "code",
- "source": "h5_headers = Presentation().fetch('h5_header')",
+ "source": "h5_headers = Presentation().to_arrays('pres_data_file')",
"id": "8cd85e91-7b20-4d8f-b410-c6be4d34d80a",
"outputs": [],
"execution_count": null
@@ -106,13 +108,15 @@
"source": [
"import numpy as np\n",
"from djimaging.utils import scanm_utils\n",
+ "from djimaging.utils.dj_storage import local_path\n",
"from tqdm.notebook import tqdm\n",
"from matplotlib import pyplot as plt\n",
"\n",
"\n",
"variances = []\n",
- "for h5_file in tqdm(h5_headers):\n",
- " ch_stacks, wparams = scanm_utils.load_stacks_from_h5(h5_file)\n",
+ "for h5_file_ref in tqdm(h5_headers):\n",
+ " with local_path(h5_file_ref, 'reference') as h5_file:\n",
+ " ch_stacks, wparams = scanm_utils.load_stacks_from_h5(h5_file)\n",
" variances.append(np.var(ch_stacks['wDataCh0'], axis=(0, 1)))\n",
"variances = np.array(variances, dtype=object)\n",
"\n",
@@ -144,8 +148,9 @@
"source": [
"fig, axs = plt.subplots(np.sum(out_i), 4, figsize=(12, np.sum(out_i)*2))\n",
"\n",
- "for i, h5_file in enumerate(h5_headers[out_i]):\n",
- " ch_stacks, wparams = scanm_utils.load_stacks_from_h5(h5_file)\n",
+ "for i, h5_file_ref in enumerate(h5_headers[out_i]):\n",
+ " with local_path(h5_file_ref, 'reference') as h5_file:\n",
+ " ch_stacks, wparams = scanm_utils.load_stacks_from_h5(h5_file)\n",
" var_i = np.var(ch_stacks['wDataCh0'], axis=(0, 1))\n",
" bad_idx = np.argmax(var_i < 100) - 2\n",
" print('(\\\"', h5_file, '\\\", ', bad_idx, ')', sep='')\n",
diff --git a/djimaging/notebooks/tutorial_my_schema.ipynb b/djimaging/notebooks/tutorial_my_schema.ipynb
index 3b1dd68..cd6e022 100644
--- a/djimaging/notebooks/tutorial_my_schema.ipynb
+++ b/djimaging/notebooks/tutorial_my_schema.ipynb
@@ -30,6 +30,9 @@
"metadata": {},
"source": [
"import os\n",
+ "os.environ[\"DJ_IGNORE_CONFIG_FILE\"] = \"true\"\n",
+ "\n",
+ "from pathlib import Path\n",
"import datajoint as dj"
],
"outputs": []
@@ -125,8 +128,7 @@
"source": [
"# Load configuration for user\n",
"dj.config.load(config_file)\n",
- "dj.config['schema_name'] = schema_name\n",
- "print(\"schema_name:\", dj.config['schema_name'])\n",
+ "print(\"schema_name:\", schema_name)\n",
"dj.conn()"
],
"outputs": []
@@ -168,7 +170,12 @@
"source": [
"from djimaging.utils.dj_utils import activate_schema\n",
"\n",
- "activate_schema(schema=schema, create_schema=True, create_tables=True)\n",
+ "activate_schema(\n",
+ " schema=schema,\n",
+ " schema_name=schema_name,\n",
+ " create_schema=True,\n",
+ " create_tables=True,\n",
+ ")\n",
"schema"
],
"outputs": []
@@ -177,7 +184,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# ERD"
+ "# Diagram"
]
},
{
@@ -189,7 +196,7 @@
"\n",
"with warnings.catch_warnings():\n",
" warnings.simplefilter(\"ignore\", FutureWarning)\n",
- " display(dj.ERD(schema))"
+ " display(dj.Diagram(schema))"
],
"outputs": []
},
@@ -205,9 +212,11 @@
"execution_count": null,
"metadata": {},
"source": [
+ "reference_root = Path(dj.config.get_store_spec('reference')['location'])\n",
+ "\n",
"userinfo = {\n",
" 'experimenter': 'TestUser', # Replace this if you want to use your own data\n",
- " 'data_dir': '/gpfs01/euler/data/Data/DataJointTestData/xy-RGCs/', # Replace this if you want to use your own data\n",
+ " 'data_dir': str(reference_root / 'Data/DataJointTestData/xy-RGCs'), # Replace the relative dataset path for your data\n",
" 'datatype_loc': 0,\n",
" 'animal_loc': 1,\n",
" 'region_loc': 2,\n",
@@ -345,7 +354,9 @@
"# To compute receptive fields, the noise stimulus trace must be loaded and set!\n",
"import h5py\n",
"\n",
- "with h5py.File(\"/gpfs01/euler/data/Resources/Stimulus/noise.h5\", \"r\") as f:\n",
+ "noise_file = reference_root / 'Resources/Stimulus/noise.h5'\n",
+ "\n",
+ "with h5py.File(noise_file, \"r\") as f:\n",
" noise_stimulus = f['stimulusarray'][:].T.astype(int)"
],
"outputs": []
@@ -358,8 +369,27 @@
"Stimulus().add_nostim(skip_duplicates=True)\n",
"Stimulus().add_chirp(spatialextent=1000, stim_name='gChirp', alias=\"chirp_gchirp_globalchirp\", skip_duplicates=True)\n",
"Stimulus().add_chirp(spatialextent=300, stim_name='lChirp', alias=\"lchirp_localchirp\", skip_duplicates=True)\n",
- "Stimulus().add_noise(stim_name='noise', pix_n_x=20, pix_n_y=15, pix_scale_x_um=30, pix_scale_y_um=30, stim_trace=noise_stimulus, skip_duplicates=True)\n",
- "Stimulus().add_movingbar(skip_duplicates=True)"
+ "Stimulus().add_noise(\n",
+ " stim_name='noise',\n",
+ " pix_n_x=20,\n",
+ " pix_n_y=15,\n",
+ " pix_scale_x_um=30,\n",
+ " pix_scale_y_um=30,\n",
+ " offset_x_um=-15,\n",
+ " offset_y_um=-15,\n",
+ " stim_path=noise_file,\n",
+ " stim_trace=noise_stimulus,\n",
+ " skip_duplicates=True,\n",
+ ")\n",
+ "Stimulus().add_movingbar(\n",
+ " ntrigger_rep=1,\n",
+ " trial_info=[dict(name=d, ntrigger=1) for d in (0, 180, 45, 225, 90, 270, 135, 315)],\n",
+ " bardx=300,\n",
+ " bardy=1000,\n",
+ " velumsec=1000,\n",
+ " tmovedurs=4.0,\n",
+ " skip_duplicates=True,\n",
+ ")"
],
"outputs": []
},
@@ -372,7 +402,7 @@
"source": [
"# Add custom stimulus if needed\n",
"# Stimulus().add_stimulus(stim_name='my_stimulus', alias=\"stimname_stimnameabbr_alternativename\", isrepeated=True, ntrigger_rep=6,\n",
- "# trial_info=[1, 2, 3, 4, 5, 6], skip_duplicates=True)"
+ "# trial_info=[dict(name=d, ntrigger=1) for d in range(1, 7)], skip_duplicates=True)"
],
"outputs": []
},
@@ -748,27 +778,9 @@
"source": [
"# RGC Classifier\n",
"\n",
- "Skip/delete this is you don't want to use the RGC classifier"
+ "Skip this section if you don't want to use the RGC classifier."
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "source": [
- "try:\n",
- " CelltypeAssignmentV2()\n",
- "except Exception as e:\n",
- " import warnings\n",
- " warnings.warn(\"\"\"\n",
- " If you want to include the RGC classifier in your database, you have to copy the respective tables from \n",
- " djimaging/schemas/rgc_classifier_schema.py \n",
- " to the schema you just imported (see above).\n",
- " Then restart the notebook and try again.\n",
- " \"\"\")"
- ],
- "outputs": []
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -781,8 +793,10 @@
"execution_count": null,
"metadata": {},
"source": [
- "if len(ClassifierV2()) == 0:\n",
- " ClassifierV2().add(classifier_file='/gpfs01/euler/data/Resources/Classifier_v2/rgc_classifier_v2.pkl', classifier_id=1)"
+ "classifier_id = 1\n",
+ "classifier_file = reference_root / 'Resources/Classifier_v2/rgc_classifier_v2_sklearn_1_7.pkl'\n",
+ "if len(ClassifierV2() & {'classifier_id': classifier_id}) == 0:\n",
+ " ClassifierV2().add(classifier_file=classifier_file, classifier_id=classifier_id)"
],
"outputs": []
},
diff --git a/djimaging/schemas/core_schema.py b/djimaging/schemas/core_schema.py
index 70353a1..64eac1f 100644
--- a/djimaging/schemas/core_schema.py
+++ b/djimaging/schemas/core_schema.py
@@ -49,7 +49,7 @@ class StackAverages(core.FieldTemplate.StackAverages):
@schema
class Stimulus(core.StimulusTemplate):
- _incl_snippet_base_dt = True
+ pass
@schema
@@ -87,6 +87,7 @@ class StackAverages(misc.HighResTemplate.StackAverages):
@schema
class RoiMask(core.RoiMaskTemplate):
_max_shift = 5 # Maximum shift of ROI mask in pixels
+ _roi_file_format = 'numpy'
field_table = Field
presentation_table = Presentation
@@ -146,24 +147,3 @@ class Averages(core.ResampledAveragesTemplate):
_norm_kind = 'amp_one' # How to normalize averages?
snippets_table = Snippets
-
-
-"""
-# Replace Snippet and Averages above if you want to use the old way of defining snippets and averages
-
-@schema
-class Snippets(core.SnippetsTemplate):
- _pad_trace = False
-
- stimulus_table = Stimulus
- presentation_table = Presentation
- traces_table = Traces
- preprocesstraces_table = PreprocessTraces
-
-
-@schema
-class Averages(core.AveragesTemplate):
- _norm_kind = 'amp_one' # How to normalize averages
-
- snippets_table = Snippets
-"""
diff --git a/djimaging/schemas/full_rgc_schema.py b/djimaging/schemas/full_rgc_schema.py
index ec8f44c..f1f02de 100644
--- a/djimaging/schemas/full_rgc_schema.py
+++ b/djimaging/schemas/full_rgc_schema.py
@@ -20,7 +20,6 @@ class ChirpFeatures(response.ChirpFeaturesRgcTemplate):
class OsDsIndexes(response.OsDsIndexesTemplate):
_reduced_storage = True
_n_shuffles = 1000
- _version = 2
stimulus_table = Stimulus
snippets_table = Snippets
@@ -51,30 +50,23 @@ class CelltypeAssignmentV2(classifier_v2.CelltypeAssignmentV2Template):
# Receptive fields
@schema
-class DNoiseTraceParams(receptivefield.DNoiseTraceParamsTemplate):
+class FastStaParams(receptivefield.FastStaParamsTemplate):
stimulus_table = Stimulus
-
-
-@schema
-class DNoiseTrace(receptivefield.DNoiseTraceTemplate):
presentation_table = Presentation
- stimulus_table = Stimulus
- traces_table = PreprocessTraces
- params_table = DNoiseTraceParams
@schema
-class STAParams(receptivefield.STAParamsTemplate):
- pass
+class FastSta(receptivefield.FastStaTemplate):
+ _traces_prefix = ''
+ params_table = FastStaParams
+ presentation_table = Presentation
+ traces_table = Traces
-@schema
-class STA(receptivefield.STATemplate):
- noise_traces_table = DNoiseTrace
- params_table = STAParams
- class DataSet(receptivefield.STATemplate.DataSet):
- pass
+@schema
+class FastStaQuality(receptivefield.FastStaQualityTemplate):
+ sta_table = FastSta
@schema
@@ -84,7 +76,7 @@ class SplitRFParams(receptivefield.SplitRFParamsTemplate):
@schema
class SplitRF(receptivefield.SplitRFTemplate):
- rf_table = STA
+ rf_table = FastSta
split_rf_params_table = SplitRFParams
diff --git a/djimaging/schemas/tutorial_schema.py b/djimaging/schemas/tutorial_schema.py
index bc26095..9cf9f5b 100644
--- a/djimaging/schemas/tutorial_schema.py
+++ b/djimaging/schemas/tutorial_schema.py
@@ -1,5 +1,5 @@
from djimaging.schemas.core_schema import *
-from djimaging.tables import misc, response, location
+from djimaging.tables import misc, response, location, classifier_v2
@schema
@@ -45,12 +45,34 @@ class ChirpFeaturesRgc(response.ChirpFeaturesRgcTemplate):
class OsDsIndexes(response.OsDsIndexesTemplate):
_reduced_storage = True
_n_shuffles = 1000
- _version = 2
stimulus_table = Stimulus
snippets_table = Snippets
+@schema
+class Baden16TracesV2(classifier_v2.Baden16TracesV2Template):
+ _stim_name_chirp = 'gChirp'
+ _stim_name_bar = 'movingbar'
+
+ traces_table = Traces
+ presentation_table = Presentation
+ stimulus_table = Stimulus
+
+
+@schema
+class ClassifierV2(classifier_v2.ClassifierV2Template):
+ pass
+
+
+@schema
+class CelltypeAssignmentV2(classifier_v2.CelltypeAssignmentV2Template):
+ classifier_table = ClassifierV2
+ baden_trace_table = Baden16TracesV2
+ field_table = Field
+ roi_table = Roi
+
+
@schema
class HighRes(misc.HighResTemplate):
raw_params_table = RawDataParams
diff --git a/djimaging/tables/_new_table_example.py b/djimaging/tables/_new_table_example.py
index 5cbd5b5..c28d60f 100644
--- a/djimaging/tables/_new_table_example.py
+++ b/djimaging/tables/_new_table_example.py
@@ -18,9 +18,9 @@ def definition(self):
# Example table that inherits primary keys from field_table
-> self.field_table
---
- table_column_a : float # Comment on number
- table_column_b : longblob # Comment on array
- table_column_c : tinyint unsigned # use this data type for True (1) or False (0)
+ table_column_a : float32 # Comment on number
+ table_column_b : # Small analysis array; use for large arrays
+ table_column_c : bool # use this data type for True (1) or False (0)
"""
return definition
diff --git a/djimaging/tables/classifier/README.md b/djimaging/tables/classifier/README.md
deleted file mode 100644
index f5b7a64..0000000
--- a/djimaging/tables/classifier/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Original version of the GCL classifier.
-This code is deprecated, please use the improved version in `djimaging/tables/classifier_v2`.
-It can be used to train a classifier, but typically you would want to use a pretrained classifier instead.
\ No newline at end of file
diff --git a/djimaging/tables/classifier/__init__.py b/djimaging/tables/classifier/__init__.py
deleted file mode 100644
index c41bb5f..0000000
--- a/djimaging/tables/classifier/__init__.py
+++ /dev/null
@@ -1,49 +0,0 @@
-from .rgc_classifier import ClassifierTemplate, ClassifierMethodTemplate, ClassifierTrainingDataTemplate
-from .celltype_assignment import CelltypeAssignmentTemplate
-from .baden16_traces import Baden16TracesTemplate
-
-"""
-DEPRECATED: Use djimaging.tables.classifier_v2 instead.
-
-Classifier for RGCs used Baden et al. 2016 dataset.
-
-Example usage:
-
-from djimaging.tables import classifier
-
-@schema
-class Baden16Traces(classifier.Baden16TracesTemplate):
- _shift_chirp = 1
- _shift_bar = -4
-
- _stim_name_chirp = 'gChirp'
- _stim_name_bar = 'movingbar'
-
- averages_table = Averages
- os_ds_table = OsDsIndexes
-
-@schema
-class ClassifierTrainingData(classifier.ClassifierTrainingDataTemplate):
- pass
-
-
-@schema
-class ClassifierMethod(classifier.ClassifierMethodTemplate):
- classifier_training_data_table = ClassifierTrainingData
-
-
-@schema
-class Classifier(classifier.ClassifierTemplate):
- classifier_training_data_table = ClassifierTrainingData
- classifier_method_table = ClassifierMethod
-
-
-@schema
-class CelltypeAssignment(classifier.CelltypeAssignmentTemplate):
- classifier_training_data_table = ClassifierTrainingData
- classifier_table = Classifier
- baden_trace_table = Baden16Traces
- field_table = Field
- roi_table = Roi
- os_ds_table = OsDsIndexes
-"""
diff --git a/djimaging/tables/classifier/baden16_traces.py b/djimaging/tables/classifier/baden16_traces.py
deleted file mode 100644
index 3a768e5..0000000
--- a/djimaging/tables/classifier/baden16_traces.py
+++ /dev/null
@@ -1,146 +0,0 @@
-"""
-Preprocess traces to match Baden et al. 2016 dataset.
-Mostly used to match traces for RGC classification.
-"""
-
-from abc import abstractmethod
-
-import numpy as np
-from matplotlib import pyplot as plt
-
-from djimaging.utils.dj_utils import get_primary_key
-from scipy import interpolate
-import datajoint as dj
-
-
-class Baden16TracesTemplate(dj.Computed):
- database = ""
-
- _shift_chirp = 1
- _shift_bar = -4 # How much the moving bar is shifted. It's actually better to use delay in the Stimulus table.
-
- _stim_name_chirp = 'gChirp'
- _stim_name_bar = 'movingbar'
-
- @property
- def definition(self):
- definition = """
- # Process traces to match Baden 2016 format, e.g. for RGC classifier.
- -> self.averages_table().proj(avg_stim_name='stim_name')
- -> self.os_ds_table().proj(os_ds_stim_name='stim_name')
- ---
- preproc_chirp: blob # preprocessed chirp trace (averaged, downsampled and normalized)
- preproc_bar: blob # preprocessed bar (time component in pref. dir., averaged and rolled)
- """
- return definition
-
- @property
- def key_source(self):
- try:
- return ((self.averages_table & dict(stim_name=self._stim_name_chirp)).proj(avg_stim_name='stim_name') *
- (self.os_ds_table & dict(stim_name=self._stim_name_bar)).proj(os_ds_stim_name='stim_name'))
- except (AttributeError, TypeError):
- pass
-
- @property
- @abstractmethod
- def averages_table(self):
- pass
-
- @property
- @abstractmethod
- def os_ds_table(self):
- pass
-
- def make(self, key):
- chirp_average, chirp_dt = (self.averages_table & dict(stim_name=self._stim_name_chirp) & key).fetch1(
- 'average', 'average_dt')
-
- bar_time_component, bar_dt = (self.os_ds_table & dict(stim_name=self._stim_name_bar) & key).fetch1(
- 'time_component', 'time_component_dt')
-
- preproc_chirp = preprocess_chirp(chirp_average, dt=chirp_dt, shift=self._shift_chirp)
- preproc_bar = preprocess_bar(bar_time_component, dt=bar_dt, shift=self._shift_bar)
-
- key = key.copy()
- key['preproc_chirp'] = preproc_chirp
- key['preproc_bar'] = preproc_bar
- self.insert1(key)
-
- def plot1(self, key=None):
- key = get_primary_key(table=self, key=key)
-
- preproc_chirp = (self & key).fetch1('preproc_chirp')
- preproc_bar = (self & key).fetch1('preproc_bar')
-
- fig, axs = plt.subplots(1, 2, figsize=(12, 3), gridspec_kw=dict(width_ratios=(3, 1)))
-
- ax = axs[0]
- ax.plot(preproc_chirp)
- ax.set(xlabel='frames', ylabel='signal', title='chirp')
-
- ax = axs[1]
- ax.plot(preproc_bar)
- ax.set(xlabel='frames', ylabel='signal', title='bar')
-
- plt.show()
-
-
-def preprocess_chirp(chirp_average, dt, shift=1):
- """
- Preprocesses chirp traces by resampling.
- The previous cutting of the last 7 frames has been replaced by an equivalent speedup factor.
- Traces are baseline corrected and normalized to be in the range [-1, 1]
- :param chirp_average: chirp average trace
- :param dt: float, time between two datapoints of chirp_average
- :param shift: int, number of frames to shift chirp trace. Old chirp was different. Shift corrects for this.
- :return: array; shape rois x frames
- """
- if chirp_average.ndim > 1:
- raise ValueError(f"chirp_trace must be 1D, but has shape {chirp_average.shape}")
-
- baden_dt = 0.128
-
- # Shift to re-correct for stimulator delay
- time_avg = np.arange(chirp_average.size) * dt + (shift * baden_dt)
-
- baden16_time = np.linspace(0, 32, 249)
-
- # Resample to Baden frequency which was (in stimulus space) slightly different
- baden16_average = interpolate.interp1d(
- time_avg, chirp_average, assume_sorted=True, bounds_error=False,
- fill_value=(chirp_average[0], chirp_average[-1]))(baden16_time)
-
- # Normalize
- baden16_average -= np.median(baden16_average[:8])
- baden16_average /= np.max(np.abs(baden16_average))
-
- return baden16_average
-
-
-def preprocess_bar(bar_average, dt, shift=-4):
- """
- Preprocesses bar time component by resampling if necessary and by rolling to match Baden traces;
- :param bar_average: moving bar average trace in preferred direction
- :param dt: float, time between two frames
- :param shift: int, number of frames to shift bar trace. Old bar has different. Shift corrects for this.
- :return: array; shape rois x frames
- """
- if bar_average.ndim > 1:
- raise ValueError(f"time_component must be 1D, but has shape {bar_average.shape}")
-
- baden_dt = 0.128
-
- time = np.arange(bar_average.size) * dt
- baden16_time = np.arange(32) * baden_dt
- baden16_average = interpolate.interp1d(
- time, bar_average, assume_sorted=True, bounds_error=False, fill_value='extrapolate')(baden16_time)
-
- # Shift
- baden16_average = np.roll(baden16_average, shift)
-
- # Subtract baseline after rolling
- baden16_average -= np.median(baden16_average[:8])
- baden16_average /= np.max(np.abs(baden16_average))
-
- return baden16_average
diff --git a/djimaging/tables/classifier/celltype_assignment.py b/djimaging/tables/classifier/celltype_assignment.py
deleted file mode 100644
index 99d4890..0000000
--- a/djimaging/tables/classifier/celltype_assignment.py
+++ /dev/null
@@ -1,551 +0,0 @@
-import pickle as pkl
-import warnings
-from abc import abstractmethod
-from typing import Mapping
-
-import datajoint as dj
-import numpy as np
-from cached_property import cached_property
-from matplotlib import pyplot as plt
-
-from djimaging.utils.baden16_utils import load_baden_data
-from djimaging.utils.dj_utils import merge_keys
-
-
-def extract_feature(
- preproc_chirp: np.ndarray,
- preproc_bar: np.ndarray,
- bar_ds_pvalue: float,
- roi_size_um2: float,
- chirp_features: np.ndarray,
- bar_features: np.ndarray,
-) -> np.ndarray:
- """Extract a feature vector for a single cell by projecting traces onto feature bases.
-
- Args:
- preproc_chirp: Preprocessed chirp trace for one cell.
- preproc_bar: Preprocessed bar trace for one cell.
- bar_ds_pvalue: Direction-selectivity p-value from the bar stimulus.
- roi_size_um2: ROI area in square micrometres.
- chirp_features: Feature basis matrix for the chirp stimulus.
- bar_features: Feature basis matrix for the bar stimulus.
-
- Returns:
- Concatenated feature vector combining chirp projections, bar projections,
- the DS p-value and ROI size.
- """
- feature_activation_chirp = np.dot(preproc_chirp, chirp_features)
- feature_activation_bar = np.dot(preproc_bar, bar_features)
-
- feature = np.concatenate(
- [feature_activation_chirp, feature_activation_bar, np.array([bar_ds_pvalue]), np.array([roi_size_um2])])
-
- return feature
-
-
-def extract_features(
- preproc_chirps: np.ndarray,
- preproc_bars: np.ndarray,
- bar_ds_pvalues: np.ndarray,
- roi_size_um2s: np.ndarray,
- chirp_features: np.ndarray,
- bar_features: np.ndarray,
-) -> np.ndarray:
- """Extract feature vectors for multiple cells by projecting traces onto feature bases.
-
- Args:
- preproc_chirps: Preprocessed chirp traces, shape ``(n_cells, n_chirp_timepoints)``.
- preproc_bars: Preprocessed bar traces, shape ``(n_cells, n_bar_timepoints)``.
- bar_ds_pvalues: Direction-selectivity p-values for each cell, shape ``(n_cells,)``.
- roi_size_um2s: ROI areas in square micrometres for each cell, shape ``(n_cells,)``.
- chirp_features: Feature basis matrix for the chirp stimulus.
- bar_features: Feature basis matrix for the bar stimulus.
-
- Returns:
- Feature matrix of shape ``(n_cells, n_features)``.
- """
- features = np.concatenate([
- np.dot(preproc_chirps, chirp_features),
- np.dot(preproc_bars, bar_features),
- bar_ds_pvalues[:, np.newaxis],
- roi_size_um2s[:, np.newaxis]
- ], axis=-1)
-
- return features
-
-
-def classify_cell(
- preproc_chirp: np.ndarray,
- preproc_bar: np.ndarray,
- bar_ds_pvalue: float,
- roi_size_um2: float,
- chirp_features: np.ndarray,
- bar_features: np.ndarray,
- classifier,
-) -> tuple:
- """Classify a single cell and return its predicted label and confidence scores.
-
- Args:
- preproc_chirp: Preprocessed chirp trace for one cell.
- preproc_bar: Preprocessed bar trace for one cell.
- bar_ds_pvalue: Direction-selectivity p-value from the bar stimulus.
- roi_size_um2: ROI area in square micrometres.
- chirp_features: Feature basis matrix for the chirp stimulus.
- bar_features: Feature basis matrix for the bar stimulus.
- classifier: Trained scikit-learn classifier with ``predict_proba`` and ``classes_``.
-
- Returns:
- A tuple ``(cell_label, confidence_scores)`` where ``cell_label`` is the predicted
- class and ``confidence_scores`` is the array of per-class probabilities.
- """
- feature = extract_feature(
- preproc_chirp, preproc_bar, bar_ds_pvalue, roi_size_um2, chirp_features, bar_features)
- confidence_scores = classifier.predict_proba(feature[np.newaxis, :]).flatten()
- cell_label = confidence_scores.argmax(axis=1)
- cell_label = classifier.classes_[cell_label][0]
- return cell_label, confidence_scores
-
-
-def classify_cells(
- preproc_chirps: np.ndarray,
- preproc_bars: np.ndarray,
- bar_ds_pvalues: np.ndarray,
- roi_size_um2s: np.ndarray,
- chirp_features: np.ndarray,
- bar_features: np.ndarray,
- classifier,
-) -> tuple:
- """Classify multiple cells and return their predicted labels and confidence scores.
-
- Args:
- preproc_chirps: Preprocessed chirp traces, shape ``(n_cells, n_chirp_timepoints)``.
- preproc_bars: Preprocessed bar traces, shape ``(n_cells, n_bar_timepoints)``.
- bar_ds_pvalues: Direction-selectivity p-values for each cell, shape ``(n_cells,)``.
- roi_size_um2s: ROI areas in square micrometres for each cell, shape ``(n_cells,)``.
- chirp_features: Feature basis matrix for the chirp stimulus.
- bar_features: Feature basis matrix for the bar stimulus.
- classifier: Trained scikit-learn classifier with ``predict_proba`` and ``classes_``.
-
- Returns:
- A tuple ``(cell_labels, confidence_scores)`` where ``cell_labels`` is an array of
- predicted classes and ``confidence_scores`` is an array of per-class probabilities,
- shape ``(n_cells, n_classes)``.
- """
- features = extract_features(
- preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s, chirp_features, bar_features)
- confidence_scores = classifier.predict_proba(features)
- cell_labels = confidence_scores.argmax(axis=1)
- cell_labels = classifier.classes_[cell_labels]
- return cell_labels, confidence_scores
-
-
-class CelltypeAssignmentTemplate(dj.Computed):
- database = ""
-
- @property
- def definition(self) -> str:
- definition = """
- # Feature-based cluster assignment of cells
- -> self.baden_trace_table
- -> self.classifier_table
- ---
- cell_label: int # predicted label with highest probability. Meaning of label depends on classifier
- max_confidence: float # confidence score for assigned cell_label, can be celltype, supergroup etc.
- confidence: blob # confidence scores (probabilities) for all celltypes
- """
- return definition
-
- @property
- def key_source(self):
- """Return the key source combining classifier and field table projections."""
- try:
- return self.classifier_table.proj() * self.field_table.proj()
- except (AttributeError, TypeError):
- pass
-
- @property
- @abstractmethod
- def classifier_training_data_table(self) -> dj.Manual:
- pass
-
- @property
- @abstractmethod
- def classifier_table(self) -> dj.Computed:
- pass
-
- @property
- @abstractmethod
- def os_ds_table(self) -> dj.Computed:
- pass
-
- @property
- @abstractmethod
- def roi_table(self) -> dj.Manual:
- pass
-
- @property
- @abstractmethod
- def baden_trace_table(self) -> dj.Computed:
- pass
-
- @property
- @abstractmethod
- def field_table(self) -> dj.Manual:
- pass
-
- @property
- def current_model_key(self):
- return self._current_model_key
-
- @current_model_key.setter
- def current_model_key(self, value: Mapping):
- if hasattr(self, "_current_model_key"):
- if value == self._current_model_key:
- pass
- else:
- print("Updating current key and invalidating cache")
- key = "classifier"
- if key in self.__dict__.keys():
- del self.__dict__[key]
- if not (self._current_model_key["training_data_hash"] == value["training_data_hash"]):
- del self.__dict__["bar_features"]
- del self.__dict__["chirp_features"]
- del self.__dict__["baden_data"]
- self._current_model_key = value
- else:
- self._current_model_key = value
-
- @cached_property
- def classifier(self):
- """Load and cache the trained classifier from its stored file path."""
- model_path = (self.classifier_table() & self.current_model_key).fetch1('classifier_file')
- with open(model_path, "rb") as f:
- model = pkl.load(f)
- return model
-
- @cached_property
- def bar_features(self) -> np.ndarray:
- """Load and cache the bar feature basis matrix from its stored file path."""
- features_bar_file = (self.classifier_training_data_table() & self.current_model_key).fetch1(
- "bar_feats_file")
- features_bar = np.load(features_bar_file)
- return features_bar
-
- @cached_property
- def chirp_features(self) -> np.ndarray:
- """Load and cache the chirp feature basis matrix from its stored file path."""
- features_chirp_file = (self.classifier_training_data_table() & self.current_model_key).fetch1(
- "chirp_feats_file")
- features_chirp = np.load(features_chirp_file)
- return features_chirp
-
- def populate(
- self, *restrictions, suppress_errors: bool = False,
- return_exception_objects: bool = False, reserve_jobs: bool = False,
- order: str = "original", limit=None, max_calls=None,
- display_progress: bool = False, processes: int = 1, make_kwargs=None,
- ) -> None:
- """Populate the table, enforcing single-process execution.
-
- Args:
- *restrictions: DataJoint restrictions passed to the parent ``populate`` call.
- suppress_errors: If True, suppress errors during population.
- return_exception_objects: If True, return exception objects instead of raising.
- reserve_jobs: If True, use the job reservation mechanism.
- order: Population order, e.g. ``"original"`` or ``"random"``.
- limit: Maximum number of keys to populate.
- max_calls: Maximum number of ``make`` calls.
- display_progress: If True, display a progress bar.
- processes: Number of parallel processes. Values greater than 1 are not
- supported and will emit a warning.
- make_kwargs: Additional keyword arguments forwarded to ``make``.
- """
- if processes > 1:
- warnings.warn('Parallel processing not implemented!')
- super().populate(
- *restrictions,
- suppress_errors=suppress_errors, return_exception_objects=return_exception_objects,
- reserve_jobs=reserve_jobs, order=order, limit=limit, max_calls=max_calls,
- display_progress=display_progress, processes=1, make_kwargs=make_kwargs)
-
- def make(self, key: dict) -> None:
- """Classify all ROIs for the given key and insert results into the table.
-
- Args:
- key: DataJoint key dictionary identifying the classifier and field to process.
- """
- roi_keys, preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s = self._fetch_data(key)
-
- if roi_keys is None:
- return
-
- cell_labels, confidence_scores = self._classify_cells(
- preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s)
-
- for roi_key, cell_label, confidence_scores_i in zip(roi_keys, cell_labels, confidence_scores):
- self.insert1(dict(**merge_keys(key, roi_key), cell_label=cell_label,
- confidence=confidence_scores_i, max_confidence=np.max(confidence_scores_i)))
-
- def _fetch_data(self, key: dict, restriction=None) -> tuple:
- """Fetch preprocessed traces and metadata for the ROIs matching ``key``.
-
- Args:
- key: DataJoint key dictionary used to restrict the query.
- restriction: Additional DataJoint restriction applied on top of ``key``.
-
- Returns:
- A tuple ``(roi_keys, preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s)``.
- Returns ``(None, None, None, None, None)`` when no matching ROIs are found.
- """
- if restriction is None:
- restriction = dict()
-
- self.current_model_key = dict(
- classifier_params_hash=key["classifier_params_hash"],
- training_data_hash=key["training_data_hash"])
-
- roi_keys = (self.baden_trace_table & key & restriction).fetch('KEY')
- if len(roi_keys) == 0:
- return None, None, None, None, None
-
- if self.os_ds_table is not None:
- data_tab = (self.baden_trace_table & key & restriction) * self.os_ds_table * self.roi_table
- else:
- data_tab = (self.baden_trace_table & key & restriction) * self.roi_table
-
- preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s = data_tab.fetch(
- 'preproc_chirp', 'preproc_bar', 'ds_pvalue', 'roi_size_um2')
-
- preproc_chirps = np.vstack(preproc_chirps)
- preproc_bars = np.vstack(preproc_bars)
-
- return roi_keys, preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s
-
- def _classify_cells(
- self,
- preproc_chirps: np.ndarray,
- preproc_bars: np.ndarray,
- bar_ds_pvalues: np.ndarray,
- roi_size_um2s: np.ndarray,
- ) -> tuple:
- """Run the cached classifier on pre-fetched feature arrays.
-
- Args:
- preproc_chirps: Preprocessed chirp traces, shape ``(n_cells, n_timepoints)``.
- preproc_bars: Preprocessed bar traces, shape ``(n_cells, n_timepoints)``.
- bar_ds_pvalues: Direction-selectivity p-values, shape ``(n_cells,)``.
- roi_size_um2s: ROI areas in square micrometres, shape ``(n_cells,)``.
-
- Returns:
- A tuple ``(cell_labels, confidence_scores)``.
- """
- cell_labels, confidence_scores = classify_cells(
- preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s,
- self.chirp_features, self.bar_features, self.classifier)
- return cell_labels, confidence_scores
-
- def plot(self, threshold_confidence: float, classifier_level: str) -> None:
- """Plot per-group histograms of predicted cell labels.
-
- Args:
- threshold_confidence: Minimum confidence required for a cell to be assigned a label.
- Cells below this threshold are shown as label ``-1``.
- classifier_level: Label level to display; one of ``'cluster'``, ``'group'``,
- or ``'super'``.
- """
- df = self.fetch(format='frame')
- groups = df.groupby(['training_data_hash', 'classifier_params_hash', 'preprocess_id'])
-
- fig, axs = plt.subplots(len(groups), 1, figsize=(12, 3 * len(groups)), squeeze=False)
- axs = axs.flatten()
-
- for ax, ((tdh, cph, pid), df_group) in zip(axs, groups):
- self._plot_key(ax=ax, df=df_group,
- classifier_params_hash=cph, training_data_hash=tdh, preprocess_id=pid,
- threshold_confidence=threshold_confidence,
- classifier_level=classifier_level)
-
- plt.tight_layout()
- plt.show()
-
- def _plot_key(self, ax, df, classifier_params_hash: str, training_data_hash: str,
- preprocess_id, threshold_confidence: float, classifier_level: str) -> None:
- """Render the cell-label histogram for a single (hash, preprocess_id) group.
-
- Args:
- ax: Matplotlib axes on which to draw.
- df: DataFrame subset for this group, as returned by ``fetch(format='frame')``.
- classifier_params_hash: Hash identifying the classifier parameter set.
- training_data_hash: Hash identifying the training data set.
- preprocess_id: Preprocessing identifier for this group.
- threshold_confidence: Cells with confidence below this value are labelled ``-1``.
- classifier_level: Label level; one of ``'cluster'``, ``'group'``, or ``'super'``.
- """
- celltypes = df.apply(
- lambda row: row['cell_label'] if row["max_confidence"] > threshold_confidence else -1, axis=1)
-
- self.current_model_key = dict(classifier_params_hash=classifier_params_hash,
- training_data_hash=training_data_hash)
- b_features, b_c_labels, b_g_labels, b_s_labels = self.classifier_training_data_table().get_training_data(
- self.current_model_key)
- if classifier_level == 'cluster':
- b_celltypes = b_c_labels
- elif classifier_level == 'group':
- b_celltypes = b_g_labels
- elif classifier_level == 'super':
- b_celltypes = b_s_labels
- else:
- raise NotImplementedError(classifier_level)
-
- ax.hist(celltypes[celltypes > 0], bins=np.arange(b_celltypes.min(), b_celltypes.max() + 1, 0.5),
- align='left')
- ax.set(ylabel='Count', xlabel='cell_label', xticks=np.arange(b_celltypes.min(), b_celltypes.max() + 1))
- ax.set_title(
- f"training_data_hash={training_data_hash}\n"
- f"classifier_params_hash={classifier_params_hash}\n"
- f"preprocess_id={preprocess_id}\n"
- f"{np.sum(celltypes == -1)}={np.mean(celltypes == -1):.0%} were not classified", loc='left')
-
- def plot_group_traces(self, threshold_confidence: float, classifier_level: str,
- celltypes=None, n_celltypes_max: int = 32,
- xlim_chirp=None, xlim_bar=None, plot_baden_data: bool = True) -> None:
- """Plot mean traces per cell type grouped by classifier and preprocessing settings.
-
- Args:
- threshold_confidence: Minimum confidence required to include a cell.
- classifier_level: Label level; one of ``'cluster'``, ``'group'``, or ``'super'``.
- celltypes: Explicit list of cell-type labels to plot. If ``None``, all
- cell types present in the data are used.
- n_celltypes_max: Maximum number of cell types to display. Excess types are
- chosen randomly.
- xlim_chirp: Optional x-axis limits for the chirp trace panel.
- xlim_bar: Optional x-axis limits for the bar trace panel.
- plot_baden_data: If ``True``, overlay traces from the Baden training data.
- """
- df = self.fetch(format='frame')
- groups = df.groupby(['training_data_hash', 'classifier_params_hash', 'preprocess_id'])
-
- for (tdh, cph, pid), df_group in groups:
- self._plot_group_traces_key(classifier_params_hash=cph, training_data_hash=tdh, preprocess_id=pid,
- threshold_confidence=threshold_confidence,
- classifier_level=classifier_level,
- celltypes=celltypes, n_celltypes_max=n_celltypes_max,
- xlim_chirp=xlim_chirp, xlim_bar=xlim_bar, plot_baden_data=plot_baden_data)
-
- def _plot_group_traces_key(self, threshold_confidence: float, classifier_level: str,
- classifier_params_hash: str, training_data_hash: str, preprocess_id,
- celltypes=None, n_celltypes_max: int = 32, xlim_chirp=None, xlim_bar=None,
- plot_baden_data: bool = True) -> None:
- """Plot mean traces per cell type for a single (hash, preprocess_id) group.
-
- Args:
- threshold_confidence: Minimum confidence required to include a cell.
- classifier_level: Label level; one of ``'cluster'``, ``'group'``, or ``'super'``.
- classifier_params_hash: Hash identifying the classifier parameter set.
- training_data_hash: Hash identifying the training data set.
- preprocess_id: Preprocessing identifier for this group.
- celltypes: Explicit list of cell-type labels to plot. If ``None``, all present
- cell types are used.
- n_celltypes_max: Maximum number of cell types to display.
- xlim_chirp: Optional x-axis limits for the chirp trace panel.
- xlim_bar: Optional x-axis limits for the bar trace panel.
- plot_baden_data: If ``True``, overlay traces from the Baden training data.
- """
- key = dict(classifier_params_hash=classifier_params_hash,
- training_data_hash=training_data_hash,
- preprocess_id=preprocess_id)
-
- # Get new data
- roi_keys, preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s = self._fetch_data(
- key=key, restriction=(self & f'max_confidence>={threshold_confidence}'))
- data_celltypes = (self & roi_keys).fetch('cell_label')
-
- # Get training data
- self.current_model_key = dict(classifier_params_hash=classifier_params_hash,
- training_data_hash=training_data_hash)
- if celltypes is not None:
- celltypes = sorted(set(celltypes).intersection(set(data_celltypes)))
- else:
- celltypes = sorted(set(data_celltypes))
-
- if n_celltypes_max is not None and len(celltypes) > n_celltypes_max:
- celltypes = sorted(np.random.choice(celltypes, n_celltypes_max, replace=False))
-
- fig, axs = plt.subplots(len(celltypes), 4,
- figsize=(12, np.minimum(2 * len(celltypes), 20)),
- sharex='col', sharey='col', squeeze=False,
- gridspec_kw=dict(width_ratios=[1, 0.6, 0.3, 0.3]))
-
- bar_ds_pvalues_bins = np.linspace(np.min(bar_ds_pvalues), np.max(bar_ds_pvalues), 51)
- roi_size_um2s_bins = np.linspace(np.min(roi_size_um2s), np.max(roi_size_um2s), 51)
-
- if plot_baden_data:
- baden_data_file = (self.classifier_training_data_table() & key).fetch1('baden_data_file')
- b_c_labels, b_g_labels, b_s_labels, b_c_traces, b_c_qi, b_mb_traces, b_mb_qi, b_mb_dsi, b_mb_dp, b_soma_um2 = \
- load_baden_data(baden_data_file)
-
- if classifier_level == 'cluster':
- b_celltypes = b_c_labels
- elif classifier_level == 'group':
- b_celltypes = b_g_labels
- elif classifier_level == 'super':
- b_celltypes = b_s_labels
- else:
- raise NotImplementedError(classifier_level)
-
- for ax_row, celltype in zip(axs, celltypes):
-
- if not np.any(data_celltypes == celltype):
- continue
-
- ax = ax_row[0]
- b_chirps = b_c_traces[b_celltypes == celltype]
- b_mean_chirp = np.mean(b_chirps, axis=0)
- ax.plot(b_chirps.T, lw=0.5, c='gray', alpha=0.5, zorder=-200)
- ax.plot(b_mean_chirp, c='k', lw=2, alpha=0.5, zorder=1)
-
- mean_chirp = np.mean(preproc_chirps[data_celltypes == celltype], axis=0)
- ax.set(title=f"cc={np.corrcoef(mean_chirp, b_mean_chirp)[0, 1]:.2f}", xlim=xlim_chirp)
-
- ax = ax_row[1]
- b_bars = b_mb_traces[b_celltypes == celltype]
- b_mean_bar = np.mean(b_bars, axis=0)
- ax.plot(b_bars.T, lw=0.5, c='gray', alpha=0.5, zorder=-200)
- ax.plot(b_mean_bar, c='k', lw=2, alpha=0.5, zorder=1)
-
- mean_bar = np.mean(preproc_bars[data_celltypes == celltype], axis=0)
- ax.set(title=f"cc={np.corrcoef(mean_bar, b_mean_bar)[0, 1]:.2f}", xlim=xlim_bar)
-
- ax = ax_row[2].twinx()
- ax.hist(b_mb_dp[b_celltypes == celltype], bar_ds_pvalues_bins, color='gray', alpha=0.5)
-
- ax = ax_row[3].twinx()
- ax.hist(b_soma_um2[b_celltypes == celltype], bins=roi_size_um2s_bins, color='gray', alpha=0.5)
-
- for ax_row, celltype in zip(axs, celltypes):
- ax_row[0].set_ylabel(celltype)
-
- if not np.any(data_celltypes == celltype):
- continue
-
- ax = ax_row[0]
- ct_chirps = preproc_chirps[data_celltypes == celltype]
- ax.plot(ct_chirps.T, lw=0.5, c='r', alpha=0.5, zorder=-100)
- ax.plot(np.mean(ct_chirps, axis=0), c='darkred', lw=2, zorder=2)
-
- ax = ax_row[1]
- ct_bars = preproc_bars[data_celltypes == celltype]
- ax.plot(ct_bars.T, lw=0.5, c='r', alpha=0.5, zorder=-100)
- ax.plot(np.mean(ct_bars, axis=0), c='darkred', lw=2, zorder=2)
-
- ax = ax_row[2]
- ax.hist(bar_ds_pvalues[data_celltypes == celltype], bins=bar_ds_pvalues_bins, color='r', alpha=0.7)
- if celltype == celltypes[0]:
- ax.set_title('bar_ds_pvalues')
-
- ax = ax_row[3]
- ax.hist(roi_size_um2s[data_celltypes == celltype], bins=roi_size_um2s_bins, color='r', alpha=0.7)
- if celltype == celltypes[0]:
- ax.set_title('roi_size_um2s')
-
- plt.tight_layout()
diff --git a/djimaging/tables/classifier/rgc_classifier.py b/djimaging/tables/classifier/rgc_classifier.py
deleted file mode 100644
index dd40f8e..0000000
--- a/djimaging/tables/classifier/rgc_classifier.py
+++ /dev/null
@@ -1,318 +0,0 @@
-import os
-import pickle as pkl
-from abc import abstractmethod
-from typing import Mapping, Dict, Any
-
-import datajoint as dj
-import numpy as np
-
-from djimaging.tables.classifier.celltype_assignment import extract_features
-from djimaging.utils.baden16_utils import load_baden_data
-from djimaging.utils.dj_utils import make_hash
-from djimaging.utils.import_helpers import extract_class_info, load_class
-
-Key = Dict[str, Any]
-
-
-def prepare_dj_config_rgc_classifier(
- output_folder: str,
- input_folder: str = "/gpfs01/euler/data/Resources/Classifier/rgc_classifier_v1",
-) -> None:
- """Configure DataJoint file stores for the RGC classifier.
-
- Sets up the ``classifier_input`` and ``classifier_output`` stores in
- ``dj.config['stores']`` and enables filepath management and native blobs.
-
- Args:
- output_folder: Path to the directory where classifier output files will be stored.
- input_folder: Path to the directory containing pre-trained classifier input files.
-
- Raises:
- AssertionError: If ``input_folder`` or ``output_folder`` do not exist on the filesystem.
- """
- stores_dict = {
- "classifier_input": {"protocol": "file", "location": input_folder, "stage": input_folder},
- "classifier_output": {"protocol": "file", "location": output_folder, "stage": output_folder},
- }
-
- # Make sure folders exits
- for store, store_dict in stores_dict.items():
- for name in store_dict.keys():
- if name in ["location", "stage"]:
- assert os.path.isdir(store_dict[name]), f'This must be a folder you have access to: {store_dict[name]}'
-
- os.environ["DJ_SUPPORT_FILEPATH_MANAGEMENT"] = "TRUE"
- dj.config['enable_python_native_blobs'] = True
-
- dj_config_stores = dj.config.get('stores', None) or dict()
- dj_config_stores.update(stores_dict)
- dj.config['stores'] = dj_config_stores
-
-
-class ClassifierTrainingDataTemplate(dj.Manual):
- database = ""
- _store = "classifier_input"
-
- @property
- def definition(self) -> str:
- definition = """
- # holds feature basis and training data for classifier
- training_data_hash : varchar(63) # hash of the classifier training data files
- ---
- output_path : varchar(191)
- baden_data_file : filepath@{store}
- chirp_feats_file : filepath@{store}
- bar_feats_file : filepath@{store}
- """.format(store=self._store)
- return definition
-
- def add_default(self, skip_duplicates: bool = False) -> None:
- """Insert the default training data entry using paths from ``dj.config['stores']``.
-
- Args:
- skip_duplicates: If ``True``, silently ignore duplicate entries.
- """
- ipath = dj.config['stores']["classifier_input"]["location"] + '/'
- opath = dj.config['stores']["classifier_output"]["location"] + '/'
-
- self.add_trainingdata(
- output_path=opath,
- baden_data_file=ipath + 'RGCData_postprocessed.mat',
- chirp_feats_file=ipath + 'chirp_feats.npz',
- bar_feats_file=ipath + 'bar_feats.npz',
- skip_duplicates=skip_duplicates,
- )
-
- def add_trainingdata(self, output_path: str, chirp_feats_file: str, bar_feats_file: str,
- baden_data_file: str, skip_duplicates: bool = False) -> None:
- key = dict(
- output_path=output_path,
- chirp_feats_file=chirp_feats_file,
- bar_feats_file=bar_feats_file,
- baden_data_file=baden_data_file
- )
- key["training_data_hash"] = make_hash(key)
- self.insert1(key, skip_duplicates=skip_duplicates)
-
- def get_training_data(self, key: Key) -> tuple:
- """Fetch and assemble the feature matrix plus label arrays for the given key.
-
- Loads the Baden reference data and feature bases from file, projects the
- reference chirp and bar traces onto those bases, and returns the resulting
- feature matrix together with cluster, group, and supergroup labels.
-
- Args:
- key: DataJoint key dictionary containing at least ``training_data_hash``.
-
- Returns:
- A tuple ``(features, b_c_labels, b_g_labels, b_s_labels)`` where
- ``features`` is an ``np.ndarray`` of shape ``(n_cells, n_features)``
- and the label arrays are 1-D ``np.ndarray`` objects.
- """
- baden_data_file, chirp_feats_file, bar_feats_file = (self & key).fetch1(
- 'baden_data_file', 'chirp_feats_file', 'bar_feats_file')
-
- b_c_labels, b_g_labels, b_s_labels, b_c_traces, b_c_qi, b_mb_traces, b_mb_qi, b_mb_dsi, b_mb_dp, b_soma_um2 = \
- load_baden_data(baden_data_file)
- chirp_features = np.load(chirp_feats_file)
- bar_features = np.load(bar_feats_file)
-
- features = extract_features(
- preproc_chirps=b_c_traces,
- preproc_bars=b_mb_traces,
- bar_ds_pvalues=b_mb_dp,
- roi_size_um2s=b_soma_um2,
- chirp_features=chirp_features,
- bar_features=bar_features,
- )
-
- return features, b_c_labels, b_g_labels, b_s_labels
-
-
-class ClassifierMethodTemplate(dj.Lookup):
- database = ""
-
- @property
- def definition(self) -> str:
- definition = """
- classifier_params_hash : varchar(63) # hash of the classifier params config
- ---
- label_kind : enum('cluster', 'group', 'super') # Predict group or cluster labels from Baden et al. 16
- classifier_fn : varchar(191) # path to classifier method fn
- classifier_config : longblob # method configuration object
- classifier_seed : int
- comment : varchar(191) # comment
- """
- return definition
-
- import_func = staticmethod(load_class)
-
- @property
- @abstractmethod
- def classifier_training_data_table(self) -> dj.Manual:
- pass
-
- def add_default(self, label_kind: str = 'group', skip_duplicates: bool = False) -> None:
- """Insert the default Random Forest classifier entry.
-
- Args:
- label_kind: Label granularity to predict; one of ``'cluster'``, ``'group'``,
- or ``'super'``.
- skip_duplicates: If ``True``, silently ignore duplicate entries.
- """
- classifier_fn = "sklearn.ensemble.RandomForestClassifier"
- classifier_config = {
- 'class_weight': 'balanced',
- 'random_state': 2001,
- 'oob_score': True,
- 'ccp_alpha': 0.00021870687842726034,
- 'max_depth': 50,
- 'max_leaf_nodes': None,
- 'min_impurity_decrease': 0,
- 'n_estimators': 1000,
- 'n_jobs': 20,
- }
-
- self.add_classifier(label_kind, classifier_fn, classifier_config, comment="Default classifier",
- skip_duplicates=skip_duplicates)
-
- def add_classifier(self, label_kind: str, classifier_fn: str, classifier_config: Mapping,
- comment: str = "", skip_duplicates: bool = False, classifier_seed: int = 42) -> None:
- if classifier_config is None:
- classifier_config = {
- 'class_weight': 'balanced',
- 'random_state': 2001,
- 'oob_score': True,
- 'ccp_alpha': 0.00021870687842726034,
- 'max_depth': 50,
- 'max_leaf_nodes': None,
- 'min_impurity_decrease': 0,
- 'n_estimators': 1000,
- 'n_jobs': 20,
- }
-
- all_config = classifier_config.copy()
- all_config['label_kind'] = label_kind
- all_config['classifier_fn'] = classifier_fn
- classifier_params_hash = make_hash(all_config)
-
- self.insert1(
- dict(
- label_kind=label_kind,
- classifier_fn=classifier_fn,
- classifier_params_hash=classifier_params_hash,
- classifier_config=classifier_config,
- classifier_seed=classifier_seed,
- comment=comment),
- skip_duplicates=skip_duplicates)
-
- def train_classifier(self, key: Key) -> tuple:
- """Train a classifier for the given key and return it together with evaluation scores.
-
- Loads training data, splits it into train/test sets, fits the classifier, evaluates
- it, then refits on all data before returning.
-
- Args:
- key: DataJoint key dictionary containing at least ``classifier_params_hash``
- and ``training_data_hash``.
-
- Returns:
- A tuple ``(classifier, score_train, score_test, score_final)`` where the scores
- are floats between 0 and 1.
- """
- from sklearn.model_selection import train_test_split
-
- label_kind, classifier_fn, classifier_config, classifier_seed = \
- (self & key).fetch1("label_kind", "classifier_fn", "classifier_config", "classifier_seed")
- classifier_config["random_state"] = classifier_seed
-
- print(f'Train classifier: {classifier_fn} {key["classifier_params_hash"]}')
-
- classifier_fn = self.import_func(*extract_class_info(classifier_fn))
- features, b_c_labels, b_g_labels, b_s_labels = self.classifier_training_data_table().get_training_data(key)
-
- if label_kind == 'cluster':
- labels = b_c_labels
- elif label_kind == 'group':
- labels = b_g_labels
- elif label_kind == 'super':
- labels = b_s_labels
- else:
- raise NotImplementedError(label_kind)
-
- print(f'Use `{label_kind}`-labels with n={np.unique(labels).size} labels')
-
- print(f'Splitting data n={labels.size} ...')
- X_train, X_test, y_train, y_test = train_test_split(
- features, labels, test_size=0.2, random_state=2001)
-
- print('Fitting classifier on training data ...')
- classifier = classifier_fn(**classifier_config)
- classifier.fit(X=X_train, y=y_train)
-
- print('Evaluate classifier on train and test data ...')
- score_train = classifier.score(X=X_train, y=y_train)
- score_test = classifier.score(X=X_test, y=y_test)
- print('Train score: {:.3f}'.format(score_train))
- print('Test score: {:.3f}'.format(score_test))
-
- print('Refitting classifier on all data ...')
- classifier = classifier_fn(**classifier_config)
- classifier.fit(X=features, y=labels)
- print('Evaluate classifier on all data ...')
- score_final = classifier.score(X=features, y=labels)
- print('Final score: {:.3f}\n'.format(score_final))
- return classifier, score_train, score_test, score_final
-
-
-class ClassifierTemplate(dj.Computed):
- database = ""
- store = "classifier_output"
-
- @property
- def definition(self) -> str:
- definition = """
- -> self.classifier_training_data_table
- -> self.classifier_method_table
- ---
- classifier_file : attach@{store}
- score_train : float # Train split data score
- score_test : float # Test split score
- score_final : float # Score after retraining on all data (the final classifier)
- """.format(store=self.store)
- return definition
-
- @property
- def key_source(self):
- """Return the key source combining method and training-data table projections."""
- try:
- return self.classifier_method_table.proj() * self.classifier_training_data_table().proj()
- except (AttributeError, TypeError):
- pass
-
- @property
- @abstractmethod
- def classifier_training_data_table(self) -> dj.Manual:
- pass
-
- @property
- @abstractmethod
- def classifier_method_table(self) -> dj.Lookup:
- pass
-
- def make(self, key: dict) -> None:
- """Train the classifier for the given key, save it to disk, and insert the result.
-
- Args:
- key: DataJoint key dictionary identifying the training data and method to use.
- """
- output_path = (self.classifier_training_data_table() & key).fetch1("output_path")
- classifier_file = os.path.join(output_path, f'rgc_classifier_{key["classifier_params_hash"]}.pkl')
- classifier, score_train, score_test, score_final = self.classifier_method_table().train_classifier(key)
-
- print(f'Saving classifier to {classifier_file}\n')
- with open(classifier_file, "wb") as f:
- pkl.dump(classifier, f)
-
- self.insert1(dict(key, classifier_file=classifier_file,
- score_train=score_train, score_test=score_test, score_final=score_final))
diff --git a/djimaging/tables/classifier_v2/baden16_traces_v2.py b/djimaging/tables/classifier_v2/baden16_traces_v2.py
index 8a0fae3..4d1c7af 100644
--- a/djimaging/tables/classifier_v2/baden16_traces_v2.py
+++ b/djimaging/tables/classifier_v2/baden16_traces_v2.py
@@ -14,12 +14,13 @@
from djimaging.tables.core.averages import compute_upsampled_average
from djimaging.tables.core.preprocesstraces import process_trace
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
import datajoint as dj
from djimaging.utils.math_utils import normalize_amp_one
from djimaging.utils.snippet_utils import split_trace_by_reps
-from djimaging.tables.response.movingbar.orientation_utils_v2 import compute_os_ds_idxs as compute_os_ds_idxs_v2
+from djimaging.tables.response.movingbar.orientation_utils import compute_os_ds_idxs
class Baden16TracesV2Template(dj.Computed):
@@ -35,15 +36,15 @@ def definition(self):
-> self.traces_table().proj(chirp_stim_name='stim_name')
-> self.traces_table().proj(bar_stim_name='stim_name')
---
- preproc_chirp: blob # preprocessed chirp trace (averaged, downsampled and normalized)
- preproc_bar: blob # preprocessed bar (time component in pref. dir., averaged and rolled)
- dir_component: blob # projection of bar trace on direction component
- ds_index: float # direction selectivity index as resulting vector length (absolute of projection on complex exponential)
- ds_pvalue: float # p-value indicating the percentile of the vector length in null distribution
- pref_dir: float # preferred direction
- os_index: float # orientation selectivity index in analogy to ds_index
- os_pvalue: float # analogous to ds_pvalue for orientation tuning
- pref_or: float # preferred orientation
+ preproc_chirp: # preprocessed chirp trace (averaged, downsampled and normalized)
+ preproc_bar: # preprocessed bar (time component in pref. dir., averaged and rolled)
+ dir_component: # projection of bar trace on direction component
+ ds_index: float32 # direction selectivity index as resulting vector length (absolute of projection on complex exponential)
+ ds_pvalue: float32 # p-value indicating the percentile of the vector length in null distribution
+ pref_dir: float32 # preferred direction
+ os_index: float32 # orientation selectivity index in analogy to ds_index
+ os_pvalue: float32 # analogous to ds_pvalue for orientation tuning
+ pref_or: float32 # preferred orientation
"""
return definition
@@ -78,15 +79,18 @@ def make(self, key):
'triggertimes')
chirp_trace, chirp_t0, chirp_dt = (self.traces_table & dict(stim_name=self._stim_name_chirp) & key).fetch1(
'trace', 'trace_t0', 'trace_dt')
+ chirp_triggertimes, chirp_trace = load_array(chirp_triggertimes), load_array(chirp_trace)
qi, chirp_average = preprocess_chirp_v2(chirp_trace, chirp_t0, chirp_dt, chirp_triggertimes, chirp_ntrigger_rep)
# Fetch bar and preprocess
- dir_order = (self.stimulus_table() & dict(stim_name=self._stim_name_bar) & key).fetch1('trial_info')
+ trial_info = (self.stimulus_table() & dict(stim_name=self._stim_name_bar) & key).fetch1('trial_info')
+ dir_order = np.asarray([trial['name'] for trial in trial_info])
bar_triggertimes = (self.presentation_table & dict(stim_name=self._stim_name_bar) & key).fetch1(
'triggertimes')
bar_trace, bar_t0, bar_dt = (self.traces_table & dict(stim_name=self._stim_name_bar) & key).fetch1(
'trace', 'trace_t0', 'trace_dt')
+ bar_triggertimes, bar_trace = load_array(bar_triggertimes), load_array(bar_trace)
bar_qi, dsi, p_dsi, pref_dir, osi, p_osi, pref_or, time_component, dir_component = preprocess_bar_v2(
bar_trace, bar_t0, bar_dt, bar_triggertimes, dir_order)
@@ -107,8 +111,8 @@ def make(self, key):
def plot1(self, key=None):
key = get_primary_key(table=self, key=key)
- preproc_chirp = (self & key).fetch1('preproc_chirp')
- preproc_bar = (self & key).fetch1('preproc_bar')
+ preproc_chirp = load_array((self & key).fetch1('preproc_chirp'))
+ preproc_bar = load_array((self & key).fetch1('preproc_bar'))
fig, axs = plt.subplots(1, 2, figsize=(12, 3), gridspec_kw=dict(width_ratios=(3, 1)))
@@ -189,7 +193,7 @@ def preprocess_bar_v2(trace, t0, dt, triggertimes, dir_order,
# Compute the preferred direction and projections
dsi, p_dsi, null_dist_dsi, pref_dir, osi, p_osi, null_dist_osi, pref_or, \
on_off, qi, time_component, dir_component, surrogate_v, dsi_s, avg_sorted_responses = \
- compute_os_ds_idxs_v2(snippets=snippets, dir_order=dir_order, dt=dt, n_shuffles=n_shuffles)
+ compute_os_ds_idxs(snippets=snippets, dir_order=dir_order, dt=dt, n_shuffles=n_shuffles)
# Normalize
baden16like_average = time_component[:32]
diff --git a/djimaging/tables/classifier_v2/celltype_assignment_v2.py b/djimaging/tables/classifier_v2/celltype_assignment_v2.py
index 498c8d2..33600c2 100644
--- a/djimaging/tables/classifier_v2/celltype_assignment_v2.py
+++ b/djimaging/tables/classifier_v2/celltype_assignment_v2.py
@@ -8,12 +8,14 @@
from djimaging.tables.classifier_v2.rgc_classifier_v2 import load_classifier_from_file, check_classifier_dict
from djimaging.utils.baden16_utils import baden_cluster_id_to_group_id, baden_group_id_to_supergroup, \
BADEN_CLUSTER_INFO, load_baden_data, baden_cluster_name_to_cluster_id
+from djimaging.utils.dj_storage import file_store_path
from djimaging.utils.dj_utils import merge_keys
class CelltypeAssignmentV2Template(dj.Computed):
database = ""
- _baden_data_file = '/gpfs01/euler/data/Resources/Classifier/rgc_classifier_v2/RGCData_postprocessed.mat'
+ _reference_store = "reference"
+ _baden_data_file = 'Resources/Classifier/rgc_classifier_v2/RGCData_postprocessed.mat'
__expected_classes = np.arange(1, 75 + 1) # Expected classes for the classifier
@property
@@ -22,14 +24,14 @@ def definition(self) -> str:
-> self.baden_trace_table
-> self.classifier_table
---
- cluster_id : tinyint unsigned # cluster ID, ranging from 1 to 75
- group_id : tinyint unsigned # group ID, ranging from 1 to 46
+ cluster_id : int32 # cluster ID, ranging from 1 to 75
+ group_id : int32 # group ID, ranging from 1 to 46
supergroup : enum('OFF', 'ON-OFF', 'Fast ON', 'Slow ON', 'Unc. ON', 'Unc. SbC', 'dAC', 'error')
- prob_cluster : float # probability of being in the given cluster
- prob_group : float # aggregated probability of being in the given group
- prob_supergroup : float # aggregated probability of being in the given supergroup
- prob_class : float # aggregated probability of being the given cell class (RGC or dAC)
- probs_per_cluster : blob # probabilities for each cluster
+ prob_cluster : float32 # probability of being in the given cluster
+ prob_group : float32 # aggregated probability of being in the given group
+ prob_supergroup : float32 # aggregated probability of being in the given supergroup
+ prob_class : float32 # aggregated probability of being the given cell class (RGC or dAC)
+ probs_per_cluster : # probabilities for each cluster
"""
return definition
@@ -64,17 +66,16 @@ def roi_table(self) -> dj.Manual:
def populate(
self,
*restrictions,
- keys=None,
suppress_errors: bool = False,
return_exception_objects: bool = False,
reserve_jobs: bool = False,
- order: str = "original",
- limit=None,
max_calls=None,
display_progress: bool = False,
processes: int = 1,
make_kwargs=None,
- ) -> None:
+ priority: int | None = None,
+ refresh: bool | None = None,
+ ) -> dict:
"""Populate the table, loading the classifier once before iterating over keys.
The classifier, chirp features, and bar features are loaded from the
@@ -83,17 +84,19 @@ def populate(
Args:
*restrictions: DataJoint restrictions; must resolve to a single classifier entry.
- keys: Optional explicit list of keys to populate.
suppress_errors: If ``True``, suppress errors during population.
return_exception_objects: If ``True``, return exception objects instead of raising.
reserve_jobs: If ``True``, use the job reservation mechanism.
- order: Population order, e.g. ``"original"`` or ``"random"``.
- limit: Maximum number of keys to populate.
max_calls: Maximum number of ``make`` calls.
display_progress: If ``True``, display a progress bar.
processes: Number of parallel processes; must be 1.
make_kwargs: Additional keyword arguments forwarded to ``make``. The keys
``'classifier'``, ``'chirp_feats'``, and ``'bar_feats'`` are reserved.
+ priority: Minimum job priority when using distributed population.
+ refresh: Whether to refresh the distributed job queue.
+
+ Returns:
+ DataJoint population summary with ``success_count`` and ``error_list``.
Raises:
NotImplementedError: If ``processes > 1``.
@@ -106,16 +109,13 @@ def populate(
"Parallel processing is not implemented for this table."
)
- if len(restrictions) == 0:
- restrictions = dict()
-
- if len(self.classifier_table & restrictions) > 1:
+ classifier_table = self.classifier_table & dj.AndList(restrictions)
+ if len(classifier_table) > 1:
raise ValueError(
"Multiple classifiers found for the given restrictions. "
"Please specify a single classifier.")
- if make_kwargs is None:
- make_kwargs = dict()
+ make_kwargs = dict(make_kwargs or {})
for key in ['classifier', 'chirp_feats', 'bar_feats']:
if key in make_kwargs:
@@ -123,7 +123,7 @@ def populate(
f"The '{key}' key is reserved and should not be provided in make_kwargs. "
"It will be automatically set based on the classifier_table.")
- classifier_file = (self.classifier_table & restrictions).fetch1('classifier_file')
+ classifier_file = classifier_table.fetch1('classifier_file')
clf_dict = load_classifier_from_file(classifier_file)
check_classifier_dict(clf_dict)
@@ -135,18 +135,17 @@ def populate(
raise ValueError("The classifier's classes do not match the expected classes.")
# populate
- super().populate(
+ return super().populate(
*restrictions,
- keys=keys,
suppress_errors=suppress_errors,
return_exception_objects=return_exception_objects,
reserve_jobs=reserve_jobs,
- order=order,
- limit=limit,
max_calls=max_calls,
display_progress=display_progress,
processes=processes,
- make_kwargs=make_kwargs
+ make_kwargs=make_kwargs,
+ priority=priority,
+ refresh=refresh,
)
def make(self, key: dict, classifier, chirp_feats: np.ndarray, bar_feats: np.ndarray) -> None:
@@ -191,8 +190,8 @@ def _fetch_data(self, key: dict, restriction=None) -> tuple:
restriction = dict()
roi_keys, preproc_chirps, preproc_bars, bar_ds_pvalues, roi_size_um2s = (
- (self.baden_trace_table & key & restriction) * self.roi_table).fetch(
- 'KEY', 'preproc_chirp', 'preproc_bar', 'ds_pvalue', 'roi_size_um2')
+ (self.baden_trace_table & key & restriction) * self.roi_table).to_arrays(
+ 'preproc_chirp', 'preproc_bar', 'ds_pvalue', 'roi_size_um2', include_key=True)
if len(roi_keys) > 0:
preproc_chirps = np.vstack(preproc_chirps)
@@ -224,7 +223,7 @@ def plot_group_traces(self, cluster_id=None, cluster_name=None, group_id=None, m
if int(group_id is not None) + int(cluster_id is not None) + int(cluster_name is not None) != 1:
raise ValueError("Provide exactly one of 'cluster_id', 'cluster_name', or 'group_id'.")
- df = self.fetch(format='frame')
+ df = self.to_pandas()
groups = df.groupby('classifier_id')
for classifier_id, df_group in groups:
@@ -279,7 +278,9 @@ def _plot_group_traces_key(self, classifier_key: dict, min_prob: float,
if plot_baden_data:
b_cis, b_gis, b_sgs, b_chirps, b_chirp_qi, b_bars, b_mb_qi, b_dsi, b_ds_pvalues, b_soma_um2s = \
- load_baden_data(self._baden_data_file, quality_filter=True)
+ load_baden_data(
+ file_store_path(self._baden_data_file, self._reference_store),
+ quality_filter=True)
if cluster_id is not None:
b_idxs = b_cis == cluster_id
@@ -368,7 +369,7 @@ def plot_type_distribution(self, min_prob: float, level: str = 'group', plot_bad
if level not in ['cluster', 'group', 'super']:
raise ValueError("Invalid level. Choose from 'cluster', 'group', or 'super'.")
- df = self.fetch(format='frame').reset_index()
+ df = self.to_pandas().reset_index()
groups = df.groupby('classifier_id')
for classifier_id, df_classifier in groups:
@@ -382,7 +383,7 @@ def _plot_type_distribution(self, df, min_prob: float, level: str, plot_baden_da
"""Render the cell-type count plot for a single classifier entry.
Args:
- df: DataFrame for one classifier, as returned by ``fetch(format='frame')``.
+ df: DataFrame for one classifier, as returned by ``to_pandas()``.
min_prob: Minimum probability threshold for including a cell.
level: Label level; one of ``'cluster'``, ``'group'``, or ``'super'``.
plot_baden_data: If ``True``, overlay counts from the Baden reference data.
@@ -403,7 +404,9 @@ def _plot_type_distribution(self, df, min_prob: float, level: str, plot_baden_da
raise NotImplementedError(f"Level '{level}' is not implemented.")
if plot_baden_data:
- b_cis, b_gis, b_sgs = load_baden_data(self._baden_data_file, quality_filter=True)[:3]
+ b_cis, b_gis, b_sgs = load_baden_data(
+ file_store_path(self._baden_data_file, self._reference_store),
+ quality_filter=True)[:3]
if level == 'cluster':
b_dist = b_cis
diff --git a/djimaging/tables/classifier_v2/rgc_classifier_v2.py b/djimaging/tables/classifier_v2/rgc_classifier_v2.py
index b507242..dbff56b 100644
--- a/djimaging/tables/classifier_v2/rgc_classifier_v2.py
+++ b/djimaging/tables/classifier_v2/rgc_classifier_v2.py
@@ -1,22 +1,28 @@
import pickle
+from pathlib import Path
import datajoint as dj
import numpy as np
+from djimaging.utils.dj_storage import file_store_path, open_object, relative_store_path
+
class ClassifierV2Template(dj.Manual):
database = ""
+ _filepath_store = "reference"
@property
def definition(self) -> str:
- definition = """
- classifier_id : int unsigned # Unique identifier for the classifier
+ definition = f"""
+ classifier_id : int64 # Unique identifier for the classifier
---
- classifier_file : varchar(255)
+ classifier_file :
"""
return definition
- def add(self, classifier_file: str = '/gpfs01/euler/data/Resources/Classifier_v2/rgc_classifier_v2.pkl',
+ def add(
+ self,
+ classifier_file: str | Path = 'Resources/Classifier_v2/rgc_classifier_v2.pkl',
classifier_id: int = 1) -> None:
"""Validate a classifier file and insert a new entry into the table.
@@ -29,15 +35,20 @@ def add(self, classifier_file: str = '/gpfs01/euler/data/Resources/Classifier_v2
:func:`check_classifier_dict`.
"""
# Test if classifier is valid
- clf_dict = load_classifier_from_file(classifier_file)
+ classifier_path = file_store_path(classifier_file, self._filepath_store)
+ classifier_store_path = relative_store_path(classifier_path, self._filepath_store)
+ clf_dict = load_classifier_from_file(classifier_path)
check_classifier_dict(clf_dict)
self.insert1(dict(
classifier_id=classifier_id,
- classifier_file=classifier_file))
+ classifier_file=classifier_store_path))
-def load_classifier_from_file(classifier_file: str) -> dict:
+def load_classifier_from_file(
+ classifier_file: str | Path | dj.ObjectRef,
+ store: str = "reference",
+) -> dict:
"""Load and return the classifier dictionary from a pickle file.
Args:
@@ -46,7 +57,7 @@ def load_classifier_from_file(classifier_file: str) -> dict:
Returns:
Dictionary containing the classifier and its associated metadata.
"""
- with open(classifier_file, 'rb') as f:
+ with open_object(classifier_file, 'rb', store=store) as f:
clf_dict = pickle.load(f)
return clf_dict
diff --git a/djimaging/tables/clustering/clustering.py b/djimaging/tables/clustering/clustering.py
index 282233a..3aaa1ee 100644
--- a/djimaging/tables/clustering/clustering.py
+++ b/djimaging/tables/clustering/clustering.py
@@ -41,6 +41,7 @@ class RoiCluster(clustering.ClusteringTemplate.RoiCluster):
import datajoint as dj
import numpy as np
from djimaging.utils import plot_utils
+from djimaging.utils.dj_storage import load_array
from matplotlib import pyplot as plt
from djimaging.utils.dj_utils import get_primary_key
@@ -52,11 +53,11 @@ class ClusteringParametersTemplate(dj.Lookup):
@property
def definition(self):
definition = """
- clustering_id: tinyint unsigned # unique param set id
+ clustering_id: int32 # unique param set id
---
kind: varchar(191)
- params_dict: longblob
- min_count: int
+ params_dict:
+ min_count: int32
"""
return definition
@@ -85,7 +86,7 @@ def definition(self):
-> self.features_table
-> self.params_table
---
- clusters : longblob
+ clusters :
"""
return definition
@@ -143,7 +144,7 @@ def definition(self):
-> master
-> master.features_table.RoiFeatures
---
- cluster_idx : int
+ cluster_idx : int32
"""
return definition
@@ -155,7 +156,7 @@ def plot1(self, key: dict = None) -> None:
"""
key = get_primary_key(table=self, key=key)
- clusters = (self & key).fetch1('clusters')
+ clusters = load_array((self & key).fetch1('clusters'))
fig, ax = plt.subplots(1, 1, figsize=(6, 2))
plot_utils.set_long_title(fig=fig, title=key)
@@ -191,7 +192,7 @@ def plot1_traces(self, key: dict = None, kind: str = 'traces') -> None:
stim_names = (self.features_table.params_table & key).fetch1('stim_names').split('_')
traces_list = (self.features_table & key).fetch1('traces')
- clusters = (self & key).fetch1('clusters')
+ clusters = load_array((self & key).fetch1('clusters'))
plot_utils.plot_clusters(traces_list, stim_names, clusters, kind=kind, title=key)
@@ -328,7 +329,7 @@ def gmm_bic_score(estimator, X_):
return -estimator.bic(X_)
def gmm_aic_score(estimator, X_):
- return -estimator.bic(X_)
+ return -estimator.aic(X_)
def gmm_loglikelihood_score(estimator, X_):
return estimator.score(X_)
diff --git a/djimaging/tables/clustering/decomposition.py b/djimaging/tables/clustering/decomposition.py
index bc9f100..91e0c21 100644
--- a/djimaging/tables/clustering/decomposition.py
+++ b/djimaging/tables/clustering/decomposition.py
@@ -21,15 +21,15 @@ class FeaturesParamsTemplate(dj.Lookup):
@property
def definition(self):
definition = """
- features_id: tinyint unsigned # unique param set id
+ features_id: int32 # unique param set id
---
kind: varchar(191)
- params_dict: longblob
- norm_trace: tinyint unsigned # Used normalized averages or averages?
+ params_dict:
+ norm_trace: bool # Used normalized averages or averages?
stim_names: varchar(191) # Stimuli to consider, separated by '_'
ncomps: varchar(191) # Number of components separated by '_'
- pre_standardize: tinyint unsigned # Standardize features before decomposition?
- post_standardize: tinyint unsigned # Standardize features after decomposition?
+ pre_standardize: bool # Standardize features before decomposition?
+ post_standardize: bool # Standardize features after decomposition?
"""
return definition
@@ -72,10 +72,10 @@ def definition(self):
definition = """
-> self.params_table
---
- features: longblob # Feature matrix.
- traces: longblob # Input traces.
- traces_reconstructed: longblob # Reconstructed traces.
- decomp_infos: longblob # information about decomposition of different stimuli
+ features: # Feature matrix.
+ traces: # Input traces.
+ traces_reconstructed: # Reconstructed traces.
+ decomp_infos: # information about decomposition of different stimuli
"""
return definition
@@ -133,13 +133,16 @@ def fetch_traces(self, key: dict, rtol: float = 0.15) -> tuple:
times = [
truncated_vstack(
- np.arange(tab.fetch(f'{stim_i}_avgs').shape[0]) * tab.fetch1(f'{stim_i}_dt')
+ np.arange(tab.to_arrays(f'{stim_i}_avgs').shape[0]) * tab.fetch1(f'{stim_i}_dt')
+ tab.fetch1(f'{stim_i}_t0'), rtol=rtol)
for stim_i in stim_names]
traces = [
- truncated_vstack(tab.fetch(f'{stim_i}_avgs'), rtol=rtol)
+ truncated_vstack(tab.to_arrays(f'{stim_i}_avgs'), rtol=rtol)
for stim_i in stim_names]
- roi_keys = tab.fetch(*self.roi_table.primary_key, as_dict=True)
+ roi_keys = [
+ {name: row[name] for name in self.roi_table.primary_key}
+ for row in tab.to_dicts()
+ ]
return traces, times, roi_keys, stim_names
def make(self, key: dict, verboselvl: int = 1) -> None:
@@ -177,7 +180,7 @@ def definition(self):
-> master
-> master.roi_table
---
- features_idx : int
+ features_idx : int32
"""
return definition
diff --git a/djimaging/tables/clustering/external_clustering.py b/djimaging/tables/clustering/external_clustering.py
index 1131443..28cc035 100644
--- a/djimaging/tables/clustering/external_clustering.py
+++ b/djimaging/tables/clustering/external_clustering.py
@@ -21,10 +21,10 @@ class RoiClusterTemplate(dj.Manual):
@property
def definition(self):
definition = """
- clustering_id : tinyint unsigned
+ clustering_id : int32
-> self.roi_table
---
- cluster_idx : int
+ cluster_idx : int32
"""
return definition
diff --git a/djimaging/tables/color_flicker/center_surround.py b/djimaging/tables/color_flicker/center_surround.py
index 21857d3..1d0bffb 100644
--- a/djimaging/tables/color_flicker/center_surround.py
+++ b/djimaging/tables/color_flicker/center_surround.py
@@ -4,6 +4,7 @@
import datajoint as dj
from matplotlib import pyplot as plt
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_trf, prep_long_title
from djimaging.utils.receptive_fields.temporal_rf_utils import compute_polarity_and_peak_idxs, \
@@ -16,10 +17,10 @@ class CenterSurroundParamsTemplate(dj.Lookup):
@property
def definition(self):
definition = """
- cs_params_id: int # unique param set id
+ cs_params_id: int32 # unique param set id
---
- peak_nstd : float # How many standard deviations does a peak need to be considered peak?
- npeaks_max : int unsigned # Maximum number of peaks, ignored if zero
+ peak_nstd : float32 # How many standard deviations does a peak need to be considered peak?
+ npeaks_max : int64 # Maximum number of peaks, ignored if zero
"""
return definition
@@ -44,14 +45,14 @@ def definition(self):
-> self.color_rf_table
-> self.cs_params_table
---
- c_polarity_idx : float # Polarity index of center RF
- c_transience_idx : float # Transience index of center RF
- c_half_amp_width : float # Half amplitude width of center RF
- c_main_peak_lag : float # Main peak lag of center RF
- s_polarity_idx : float # Polarity index of surround RF
- s_transience_idx : float # Transience index of surround RF
- s_half_amp_width : float # Half amplitude width of surround RF
- s_main_peak_lag : float # Main peak lag of surround RF
+ c_polarity_idx : float32 # Polarity index of center RF
+ c_transience_idx : float32 # Transience index of center RF
+ c_half_amp_width : float32 # Half amplitude width of center RF
+ c_main_peak_lag : float32 # Main peak lag of center RF
+ s_polarity_idx : float32 # Polarity index of surround RF
+ s_transience_idx : float32 # Transience index of surround RF
+ s_half_amp_width : float32 # Half amplitude width of surround RF
+ s_main_peak_lag : float32 # Main peak lag of surround RF
"""
return definition
@@ -67,6 +68,11 @@ def key_source(self):
def color_rf_table(self):
pass
+ @property
+ @abstractmethod
+ def color_rf_time_table(self):
+ pass
+
@property
@abstractmethod
def cs_params_table(self):
@@ -85,10 +91,10 @@ def fetch1_rf_time(self, key: dict) -> np.ndarray:
1-D array of RF time values.
"""
try:
- rf_time = (self.color_rf_table & key).fetch1('rf_time')
+ rf_time = (self.color_rf_time_table & key).fetch1('rf_time')
except dj.DataJointError:
- rf_time = (self.color_rf_table & key).fetch1('model_dict')['rf_time']
- return rf_time
+ rf_time = (self.color_rf_time_table & key).fetch1('model_dict')['rf_time']
+ return load_array(rf_time)
def make(self, key: dict, plot: bool = False) -> None:
"""Compute center/surround RF metrics and insert them into the table.
@@ -100,7 +106,11 @@ def make(self, key: dict, plot: bool = False) -> None:
interactive debugging.
"""
rf_time = self.fetch1_rf_time(key=key)
- rf = (self.color_rf_table & key).fetch1('rf')
+ rf = load_array((self.color_rf_table & key).fetch1('rf')).squeeze()
+
+ if rf.ndim != 2:
+ raise ValueError(f"rf must be 2d (time x center/surround), but got shape {rf.shape}")
+
peak_nstd, npeaks_max = (self.cs_params_table & key).fetch1('peak_nstd', 'npeaks_max')
polarity_idxs = []
@@ -139,7 +149,7 @@ def plot(self) -> tuple:
"""
fig, axs = plt.subplots(1, 3, figsize=(12, 3))
for ax, name in zip(axs, ['transience_idx', 'half_amp_width', 'main_peak_lag']):
- ax.hist(self.fetch(name))
+ ax.hist(self.to_arrays(name))
ax.set_title(name)
plt.tight_layout()
return fig, axs
@@ -156,7 +166,7 @@ def plot1(self, key: dict = None) -> tuple:
key = get_primary_key(table=self, key=key)
rf_time = self.fetch1_rf_time(key=key)
- rf = (self.color_rf_table & key).fetch1('rf')
+ rf = load_array((self.color_rf_table & key).fetch1('rf')).squeeze()
peak_nstd, npeaks_max = (self.cs_params_table & key).fetch1('peak_nstd', 'npeaks_max')
diff --git a/djimaging/tables/core/__init__.py b/djimaging/tables/core/__init__.py
index c0ad7d5..524bd1b 100644
--- a/djimaging/tables/core/__init__.py
+++ b/djimaging/tables/core/__init__.py
@@ -1,4 +1,4 @@
-from .averages import AveragesTemplate, ResampledAveragesTemplate
+from .averages import ResampledAveragesTemplate
from .experiment import ExperimentTemplate
from .preprocesstraces import PreprocessTracesTemplate, PreprocessParamsTemplate
from .snippets import SnippetsTemplate, GroupSnippetsTemplate
@@ -10,4 +10,4 @@
from .roi_mask import RoiMaskTemplate
from .roi import RoiTemplate
from .traces import TracesTemplate
-from .stim_logs import QdsPyLogFileTemplate, QdsPyLogTemplate
+from .stim_logs import QdsPyLogFileTemplate, QdsPyLogTemplate, PresentationLogTemplate
diff --git a/djimaging/tables/core/averages.py b/djimaging/tables/core/averages.py
index 4655c46..decd731 100644
--- a/djimaging/tables/core/averages.py
+++ b/djimaging/tables/core/averages.py
@@ -5,14 +5,27 @@
import numpy as np
from matplotlib import pyplot as plt
-from djimaging.tables.core.snippets import get_aligned_snippets_times
+from djimaging.tables.core.snippets import fetch_snippets_and_times
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils import plot_utils, math_utils, trace_utils
-class AveragesTemplate(dj.Computed):
+class ResampledAveragesTemplate(dj.Computed):
+ """Averages of resampled snippets
+
+ Example usage:
+
+ @schema
+ class ResampledAverages(core.ResampledAveragesTemplate):
+ _norm_kind = 'amp_one'
+ _f_resample = 500
+ snippets_table = Snippets
+ """
+
database = ""
_norm_kind = 'amp_one'
+ _f_resample = 60
@property
def definition(self):
@@ -20,19 +33,14 @@ def definition(self):
# Averages of snippets
-> self.snippets_table
---
- average :longblob # array of snippet average (time)
- average_norm :longblob # normalized array of snippet average (time)
- average_t0 :float # time of the first sample of the average
- average_dt :float # time between samples of the average
- triggertimes_rel :longblob # array of relative triggertimes
+ average : # array of snippet average (time)
+ average_norm : # normalized array of snippet average (time)
+ average_t0 :float32 # time of the first sample of the average
+ average_dt :float32 # time between samples of the average
+ triggertimes_rel : # array of relative triggertimes
"""
return definition
- @property
- @abstractmethod
- def snippets_table(self):
- pass
-
@property
def key_source(self):
try:
@@ -71,82 +79,6 @@ def normalize_average(self, average: np.ndarray) -> np.ndarray:
return average_norm
- def make(self, key: dict) -> None:
- """Compute and store the average of snippets for a given key.
-
- Fetches snippets from the snippets table, aligns them in time,
- computes the mean across repetitions, normalizes the average, and
- inserts the result into this table.
-
- Parameters
- ----------
- key : dict
- The primary key identifying the entry to populate.
- """
- snippets_t0, snippets_dt, snippets = (self.snippets_table() & key).fetch1(
- 'snippets_t0', 'snippets_dt', 'snippets')
- triggertimes_snippets = (self.snippets_table() & key).fetch1('triggertimes_snippets')
-
- if snippets.shape[1] <= 1:
- warnings.warn(f"Skipping {key} because it has only one repetition.")
- return
-
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
- average_times = get_aligned_snippets_times(snippets_times=snippets_times)
-
- average = np.mean(snippets, axis=1)
-
- average_norm = self.normalize_average(average)
-
- triggertimes_rel = np.mean(triggertimes_snippets - triggertimes_snippets[0, :], axis=1)
-
- self.insert1(dict(
- **key,
- average=average.astype(np.float32),
- average_norm=average_norm.astype(np.float32),
- average_t0=average_times[0],
- average_dt=snippets_dt,
- triggertimes_rel=triggertimes_rel.astype(np.float32),
- ))
-
- def plot1(self, key: dict | None = None, xlim: tuple | None = None) -> None:
- """Plot snippets and average trace for a single entry.
-
- Parameters
- ----------
- key : dict | None, optional
- Primary key identifying the entry to plot. If None, the first
- available key is used.
- xlim : tuple | None, optional
- x-axis limits for the plot. Default is None (auto).
- """
- key = get_primary_key(table=self, key=key)
-
- snippets_t0, snippets_dt, snippets = (self.snippets_table & key).fetch1(
- 'snippets_t0', 'snippets_dt', 'snippets')
-
- average, average_norm, average_t0, average_dt, triggertimes_rel = \
- (self & key).fetch1('average', 'average_norm', 'average_t0', 'average_dt', 'triggertimes_rel')
-
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
-
- average_times = get_aligned_snippets_times(snippets_times=snippets_times)
-
- fig, axs = plt.subplots(2, 1, figsize=(10, 4), sharex='all')
-
- aligned_times = get_aligned_snippets_times(snippets_times=snippets_times)
- plot_utils.plot_traces(
- ax=axs[0], time=aligned_times, traces=snippets.T)
- axs[0].set(ylabel='trace', xlabel='aligned time')
-
- plot_utils.plot_trace_and_trigger(
- ax=axs[1], time=average_times, trace=average,
- triggertimes=triggertimes_rel, trace_norm=average_norm)
-
- plt.show()
-
def plot(self, restriction: dict | None = None, sort: bool = True) -> None:
"""Plot heatmaps of all averages matching the given restriction.
@@ -161,8 +93,8 @@ def plot(self, restriction: dict | None = None, sort: bool = True) -> None:
if restriction is None:
restriction = dict()
- averages = (self & restriction).fetch('average')
- averages_norm = (self & restriction).fetch('average_norm')
+ averages = (self & restriction).to_arrays('average')
+ averages_norm = (self & restriction).to_arrays('average_norm')
averages = math_utils.padded_vstack(averages, cval=np.nan)
averages_norm = math_utils.padded_vstack(averages_norm, cval=np.nan)
@@ -181,23 +113,6 @@ def plot(self, restriction: dict | None = None, sort: bool = True) -> None:
plot_utils.plot_signals_heatmap(ax=axs[1], signals=averages_norm[sort_idxs, :])
plt.show()
-
-class ResampledAveragesTemplate(AveragesTemplate):
- """Averages of resampled snippets
-
- Example usage:
-
- @schema
- class ResampledAverages(core.ResampledAveragesTemplate):
- _norm_kind = 'amp_one'
- _f_resample = 500
- snippets_table = Snippets
- """
-
- database = ""
- _norm_kind = 'amp_one'
- _f_resample = 60
-
@property
@abstractmethod
def snippets_table(self):
@@ -215,18 +130,12 @@ def make(self, key: dict) -> None:
key : dict
The primary key identifying the entry to populate.
"""
- snippets_t0, snippets_dt, snippets = (self.snippets_table() & key).fetch1(
- 'snippets_t0', 'snippets_dt', 'snippets')
+ snippets, snippets_times, triggertimes_snippets = fetch_snippets_and_times(self.snippets_table, key)
if snippets.shape[1] <= 1:
warnings.warn(f"Skipping {key} because it has only one repetition.")
return
- triggertimes_snippets = (self.snippets_table() & key).fetch1('triggertimes_snippets')
-
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
-
average, average_times, _ = compute_upsampled_average(
snippets, snippets_times, triggertimes_snippets, f_resample=self._f_resample)
average_norm = self.normalize_average(average)
@@ -254,14 +163,13 @@ def plot1(self, key: dict | None = None, xlim: tuple | None = None) -> None:
"""
key = get_primary_key(table=self, key=key)
- snippets_t0, snippets_dt, snippets, triggertimes_snippets = (self.snippets_table & key).fetch1(
- 'snippets_t0', 'snippets_dt', 'snippets', 'triggertimes_snippets')
+ snippets, snippets_times, triggertimes_snippets = fetch_snippets_and_times(self.snippets_table, key)
average, average_norm, average_t0, average_dt, triggertimes_rel = \
(self & key).fetch1('average', 'average_norm', 'average_t0', 'average_dt', 'triggertimes_rel')
+ average, average_norm, triggertimes_rel = (
+ load_array(average), load_array(average_norm), load_array(triggertimes_rel))
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
average_times = np.arange(len(average)) * average_dt + average_t0
fig, axs = plt.subplots(2, 1, figsize=(10, 4), sharex='all')
diff --git a/djimaging/tables/core/experiment.py b/djimaging/tables/core/experiment.py
index 2092167..e8c5723 100644
--- a/djimaging/tables/core/experiment.py
+++ b/djimaging/tables/core/experiment.py
@@ -20,9 +20,9 @@ def definition(self):
# Header-File location, name and path
-> self.userinfo_table
date :date # date of recording
- exp_num :tinyint unsigned # experiment number in a day
+ exp_num :int32 # experiment number in a day
---
- header_path :varchar(191) # path to header file
+ header_path :varchar(191) # experiment directory containing the header file
header_name :varchar(63) # name of header file
"""
return definition
@@ -76,7 +76,7 @@ def rescan_filesystem(self, restrictions: dict = None, verboselvl: int = 1, supp
if restrictions is None:
restrictions = dict()
- for key in (self.key_source & restrictions).fetch(as_dict=True):
+ for key in (self.key_source & restrictions).to_dicts():
if verboselvl > 0:
print(f"Scanning for experimenter: {key['experimenter']}")
@@ -327,19 +327,19 @@ def definition(self):
# General preparation details set by user in preprocessing
-> master
---
- eye :enum("left", "right", "unknown") # left or right eye of the animal
+ eye :enum('left', 'right', 'unknown') # left or right eye of the animal
projname :varchar(191) # name of experimental project
setupid :varchar(191) # setup 1-3
- prep="wholemount" :enum("wholemount", "slice") # preparation type of the retina
+ prep="wholemount" :enum('wholemount', 'slice') # preparation type of the retina
preprem :varchar(191) # comments on the preparation
darkadapt_hrs :varchar(191) # time spent dark adapting animal before disection
slicethickness_um :varchar(191) # thickness of each slice in slice preparation
bathtemp_degc :varchar(191) # temperature of bath chamber
- prepwmorient :smallint # retina orientation in chamber (0° = dorsal away from experimenter). Defaults to 0. Use -1, 111 or "unkown" to encode unknown.
- odx :float # x location of optic disk as read in from .ini file
- ody :float # y location of optic disk as read in from .ini file (if available)
- odz :float # z location of optic disk as read in from .ini file (if available)
- od_ini_flag :tinyint unsigned # flag (0, 1) indicating whether (1) or whether not (0)
+ prepwmorient :int32 # retina orientation in chamber (0° = dorsal away from experimenter). Defaults to 0. Use -1, 111 or "unkown" to encode unknown.
+ odx :float32 # x location of optic disk as read in from .ini file
+ ody :float32 # y location of optic disk as read in from .ini file (if available)
+ odz :float32 # z location of optic disk as read in from .ini file (if available)
+ od_ini_flag :bool # flag (0, 1) indicating whether (1) or whether not (0)
# the optic disk position was documented in .ini file and
# is valid to use
"""
@@ -393,7 +393,7 @@ def definition(self):
# Pharmacology Info
-> master
---
- pharmaflag :tinyint unsigned # 1 there was pharma, 0 no pharma
+ pharmaflag :bool # 1 there was pharma, 0 no pharma
drug :varchar(191) # which drug was applied
pharmconc :varchar(191) # concentration used in micromolar
preapptime :varchar(191) # preapplication time
diff --git a/djimaging/tables/core/field.py b/djimaging/tables/core/field.py
index 3c0b973..4dfe025 100644
--- a/djimaging/tables/core/field.py
+++ b/djimaging/tables/core/field.py
@@ -8,6 +8,7 @@
import numpy as np
import pandas as pd
+from djimaging.utils.dj_storage import load_array, relative_store_path
from djimaging.utils.filesystem_utils import get_file_info_df
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_field
@@ -16,6 +17,7 @@
class FieldTemplate(dj.Computed):
database = ""
+ _filepath_store = "reference"
incl_region = True # Include region as primary key?
incl_cond1 = False # Include condition 1 as primary key?
incl_cond2 = False # Include condition 2 as primary key?
@@ -39,21 +41,21 @@ def definition(self):
if self.incl_cond3:
definition += " cond3 :varchar(16) # condition (pharmacological or other)\n"
- definition += """
+ definition += f"""
---
- field_data_file: varchar(191) # info extracted from which file?
- absx: float # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
- absy: float # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
- absz: float # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
- scan_type: enum("xy", "xz", "xyz") # Type of scan
- npixartifact : int unsigned # Number of pixel with light artifact
- nxpix: int unsigned # number of pixels in x
- nypix: int unsigned # number of pixels in y
- nzpix: int unsigned # number of pixels in z
- nxpix_offset: int unsigned # number of offset pixels in x
- nxpix_retrace: int unsigned # number of retrace pixels in x
- pixel_size_um :float # width of a pixel in um (also height if y is second dimension)
- z_step_um = NULL :float # z-step in um
+ field_data_file: # source acquisition file
+ absx: float32 # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
+ absy: float32 # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
+ absz: float32 # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
+ scan_type: enum('xy', 'xz', 'xyz') # Type of scan
+ npixartifact : int64 # Number of pixel with light artifact
+ nxpix: int64 # number of pixels in x
+ nypix: int64 # number of pixels in y
+ nzpix: int64 # number of pixels in z
+ nxpix_offset: int64 # number of offset pixels in x
+ nxpix_retrace: int64 # number of retrace pixels in x
+ pixel_size_um :float32 # width of a pixel in um (also height if y is second dimension)
+ z_step_um = NULL :float32 # z-step in um
"""
return definition
@@ -107,7 +109,7 @@ def definition(self):
-> master
ch_name : varchar(32) # name of the channel
---
- ch_average :longblob # Stack median over time
+ ch_average : # Stack median over time
"""
return definition
@@ -379,8 +381,7 @@ def _load_first_file_wo_error(
return rec
- @staticmethod
- def complete_keys(base_key: dict, rec: ScanMRecording) -> tuple[dict, list]:
+ def complete_keys(self, base_key: dict, rec: ScanMRecording) -> tuple[dict, list]:
"""Build the field entry dict and stack-average entry dicts from a recording.
Parameters
@@ -400,7 +401,7 @@ def complete_keys(base_key: dict, rec: ScanMRecording) -> tuple[dict, list]:
"""
field_entry = deepcopy(base_key)
- field_entry["field_data_file"] = rec.filepath
+ field_entry["field_data_file"] = relative_store_path(rec.filepath, self._filepath_store)
field_entry["absx"] = rec.pos_x_um
field_entry["absy"] = rec.pos_y_um
field_entry["absz"] = rec.pos_z_um
@@ -437,9 +438,10 @@ def plot1(self, key: dict | None = None, gamma: float = 0.7) -> None:
"""
key = get_primary_key(table=self, key=key)
data_name, alt_name = (self.userinfo_table & key).fetch1('data_stack_name', 'alt_stack_name')
- main_ch_average = (self.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
+ main_ch_average = load_array((self.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
try:
- alt_ch_average = (self.StackAverages & key & f'ch_name="{alt_name}"').fetch1('ch_average')
+ alt_ch_average = load_array(
+ (self.StackAverages & key & dict(ch_name=alt_name)).fetch1('ch_average'))
except dj.DataJointError:
alt_ch_average = np.full_like(main_ch_average, np.nan)
diff --git a/djimaging/tables/core/preprocesstraces.py b/djimaging/tables/core/preprocesstraces.py
index 7d24324..d663d4f 100644
--- a/djimaging/tables/core/preprocesstraces.py
+++ b/djimaging/tables/core/preprocesstraces.py
@@ -2,6 +2,7 @@
import warnings
from abc import abstractmethod
+from collections.abc import Callable
import datajoint as dj
import numpy as np
@@ -9,6 +10,7 @@
from scipy import signal
from djimaging.utils import filter_utils, math_utils, plot_utils, trace_utils
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.filter_utils import lowpass_filter_trace
from djimaging.utils.plot_utils import plot_trace_and_trigger, prep_long_title
@@ -21,15 +23,15 @@ class PreprocessParamsTemplate(dj.Lookup):
def definition(self):
definition = """
-> self.stimulus_table
- preprocess_id: tinyint unsigned # unique param set id
+ preprocess_id: int32 # unique param set id
---
- window_length: int # window length for SavGol filter in seconds
- poly_order: int # order of polynomial for savgol filter
- non_negative: tinyint unsigned # Clip negative values of trace
- subtract_baseline: tinyint unsigned # Subtract baseline
- standardize: tinyint unsigned # standardize (1: with sd of baseline, 2: sd of trace, 0: nothing)
- f_cutoff = 0 : float # Cutoff frequency for low pass filter, only applied when > 0.
- fs_resample = 0 : float # Resampling frequency, only applied when > 0.
+ window_length: int32 # window length for SavGol filter in seconds
+ poly_order: int32 # order of polynomial for savgol filter
+ non_negative: bool # Clip negative values of trace
+ subtract_baseline: bool # Subtract baseline
+ standardize: int32 # standardize (1: with sd of baseline, 2: sd of trace, 0: nothing)
+ f_cutoff = 0 : float32 # Cutoff frequency for low pass filter, only applied when > 0.
+ fs_resample = 0 : float32 # Resampling frequency, only applied when > 0.
"""
return definition
@@ -82,7 +84,7 @@ def add_default(
If True, silently skip duplicate entries. Default is False.
"""
if stim_names is None:
- stim_names = (self.stimulus_table()).fetch('stim_name')
+ stim_names = (self.stimulus_table()).to_arrays('stim_name')
key = dict(
preprocess_id=preprocess_id,
@@ -113,10 +115,10 @@ def definition(self):
-> self.traces_table
-> self.preprocessparams_table
---
- pp_trace: longblob # preprocessed trace
- smoothed_trace: longblob # output of savgol filter which is subtracted from the raw trace
- pp_trace_t0: float # numerical array of trace times
- pp_trace_dt: float # time between frames
+ pp_trace: # preprocessed trace
+ smoothed_trace: # output of savgol filter which is subtracted from the raw trace
+ pp_trace_t0: float32 # numerical array of trace times
+ pp_trace_dt: float32 # time between frames
"""
return definition
@@ -165,7 +167,8 @@ def make(self, key: dict) -> None:
'f_cutoff', 'fs_resample')
trace, trace_t0, trace_dt = (self.traces_table() & key).fetch1('trace', 'trace_t0', 'trace_dt')
- triggertimes = (self.presentation_table() & key).fetch1('triggertimes')
+ trace = load_array(trace)
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
stim_start = triggertimes[0] if len(triggertimes) > 0 else None
@@ -201,31 +204,14 @@ def gui_clip_trace(self, key: dict) -> None:
trace, smoothed_trace, trace_t0, trace_dt = (self & key).fetch1(
'pp_trace', 'smoothed_trace', 'pp_trace_t0', 'pp_trace_dt')
- trace_t = np.arange(trace.size) * trace_dt + trace_t0
+ trace, smoothed_trace = load_array(trace), load_array(smoothed_trace)
- import ipywidgets as widgets
+ def save_clip(i0: int, i1: int) -> None:
+ self.update_key_from_gui(
+ key, trace=trace[i0:i1], smoothed_trace=smoothed_trace[i0:i1],
+ trace_t0=trace_t0 + i0 * trace_dt, trace_dt=trace_dt)
- w_left = widgets.IntSlider(0, min=0, max=trace.size - 1, step=1,
- layout=widgets.Layout(width='800px'))
- w_right = widgets.IntSlider(trace.size - 1, min=0, max=trace.size - 1, step=1,
- layout=widgets.Layout(width='800px'))
- w_save = widgets.Checkbox(False)
-
- title = 'Not saved\n' + prep_long_title(key)
-
- @widgets.interact(left=w_left, right=w_right, save=w_save)
- def plot_fit(left=0, right=trace.size - 1, save=False):
- nonlocal title
-
- plot_left_right_clipping(trace, trace_t, left, right, title)
-
- if save:
- i0, i1 = (right, left + 1) if right < left else (left, right + 1)
- self.update_key_from_gui(
- key, trace=trace[i0:i1], smoothed_trace=smoothed_trace[i0:i1],
- trace_t0=trace_t0 + left * trace_dt, trace_dt=trace_dt)
- title = f'SAVED: left={left}, right={right}\n{prep_long_title(key)}'
- w_save.value = False
+ gui_clip_trace(trace, trace_t0, trace_dt, key, save_clip)
def update_key_from_gui(
self,
@@ -271,7 +257,9 @@ def plot1(self, key: dict | None = None, xlim: tuple | None = None, ylim: tuple
pp_trace_t0, pp_trace_dt, pp_trace, smoothed_trace = (self & key).fetch1(
"pp_trace_t0", "pp_trace_dt", "pp_trace", "smoothed_trace")
trace_t0, trace_dt, trace = (self.traces_table() & key).fetch1("trace_t0", "trace_dt", "trace")
- triggertimes = (self.presentation_table() & key).fetch1("triggertimes")
+ pp_trace, smoothed_trace = load_array(pp_trace), load_array(smoothed_trace)
+ trace = load_array(trace)
+ triggertimes = load_array((self.presentation_table() & key).fetch1("triggertimes"))
trace_times = np.arange(trace.size) * trace_dt + pp_trace_t0
pp_trace_times = np.arange(pp_trace.size) * pp_trace_dt + pp_trace_t0
@@ -309,7 +297,7 @@ def plot(self, restriction: dict | None = None, sort: bool = True) -> None:
if restriction is None:
restriction = dict()
- preprocess_traces = (self & restriction).fetch("pp_trace")
+ preprocess_traces = (self & restriction).to_arrays("pp_trace")
preprocess_traces = math_utils.padded_vstack(preprocess_traces, cval=np.nan)
n = preprocess_traces.shape[0]
@@ -656,6 +644,36 @@ def process_trace(
return trace, smoothed_trace, trace_dt_new
+def gui_clip_trace(
+ trace: np.ndarray,
+ trace_t0: float,
+ trace_dt: float,
+ key: dict,
+ save_clip: Callable[[int, int], None],
+) -> None:
+ """Show clip controls; call save_clip(start, stop) with slice bounds on save."""
+ import ipywidgets as widgets
+
+ trace_t = np.arange(trace.size) * trace_dt + trace_t0
+ w_left = widgets.IntSlider(0, min=0, max=trace.size - 1, step=1,
+ layout=widgets.Layout(width='800px'))
+ w_right = widgets.IntSlider(trace.size - 1, min=0, max=trace.size - 1, step=1,
+ layout=widgets.Layout(width='800px'))
+ w_save = widgets.Checkbox(False)
+ title = 'Not saved\n' + prep_long_title(key)
+
+ @widgets.interact(left=w_left, right=w_right, save=w_save)
+ def plot_fit(left: int = 0, right: int = trace.size - 1, save: bool = False) -> None:
+ nonlocal title
+ plot_left_right_clipping(trace, trace_t, left, right, title)
+
+ if save:
+ i0, i1 = min(left, right), max(left, right) + 1
+ save_clip(i0, i1)
+ title = f'SAVED: left={left}, right={right}\n{prep_long_title(key)}'
+ w_save.value = False
+
+
def plot_left_right_clipping(
trace_: np.ndarray,
trace_t_: np.ndarray,
diff --git a/djimaging/tables/core/presentation.py b/djimaging/tables/core/presentation.py
index 9fa137d..4b37384 100644
--- a/djimaging/tables/core/presentation.py
+++ b/djimaging/tables/core/presentation.py
@@ -8,6 +8,7 @@
import matplotlib.pyplot as plt
import numpy as np
+from djimaging.utils.dj_storage import load_array, local_path, relative_store_path
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_field
from djimaging.utils.scanm.recording import ScanMRecording
@@ -15,6 +16,7 @@
class PresentationTemplate(dj.Computed):
database = ""
+ _filepath_store = "reference"
incl_region = True # Include region as primary key?
incl_cond1 = True # Include condition 1 as primary key?
incl_cond2 = False # Include condition 2 as primary key?
@@ -38,24 +40,24 @@ def definition(self):
if self.incl_cond3 and not self.field_table.incl_cond3:
definition += " cond3 :varchar(16) # condition (pharmacological or other)\n"
- definition += """
+ definition += f"""
---
- pres_data_file :varchar(191) # path to file (e.g. h5 file)
- triggertimes :longblob # triggertimes in each presentation
- trigger_valid :tinyint unsigned # Are triggers as expected (1) or not (0)?
- absx: float # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
- absy: float # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
- absz: float # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
- scan_type: enum("xy", "xz", "xyz") # Type of scan
- npixartifact : int unsigned # number of pixel with light artifact
- nxpix: int unsigned # number of pixels in x
- nypix: int unsigned # number of pixels in y
- nzpix: int unsigned # number of pixels in z
- nxpix_offset: int unsigned # number of offset pixels in x
- nxpix_retrace: int unsigned # number of retrace pixels in x
- pixel_size_um :float # width of a pixel in um (also height if y is second dimension)
- z_step_um = NULL :float # z-step in um
- nframes: int unsigned # number of pixels in time
+ pres_data_file : # source acquisition file (e.g. HDF5)
+ triggertimes : # triggertimes in each presentation
+ trigger_valid :bool # Are triggers as expected (1) or not (0)?
+ absx: float32 # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
+ absy: float32 # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
+ absz: float32 # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
+ scan_type: enum('xy', 'xz', 'xyz') # Type of scan
+ npixartifact : int64 # number of pixel with light artifact
+ nxpix: int64 # number of pixels in x
+ nypix: int64 # number of pixels in y
+ nzpix: int64 # number of pixels in z
+ nxpix_offset: int64 # number of offset pixels in x
+ nxpix_retrace: int64 # number of retrace pixels in x
+ pixel_size_um :float32 # width of a pixel in um (also height if y is second dimension)
+ z_step_um = NULL :float32 # z-step in um
+ nframes: int64 # number of pixels in time
"""
return definition
@@ -119,7 +121,7 @@ def definition(self):
-> master
ch_name : varchar(32) # name of the channel
---
- ch_average : longblob # Stack median over time
+ ch_average : # Stack median over time
"""
return definition
@@ -130,13 +132,13 @@ def definition(self):
# Data read from wParamsNum and wParamsStr tables in h5 file
-> master
---
- scan_frequency=-1 :float # Scanning frequency in Hz
- scan_period=-1 :float # Scanning duration per frame in seconds
- line_duration=-1 :float # Line duration from OS_Parameters
- real_pixel_duration=-1 :float # Duration of a pixel (un-cropped)
- zoom :float # Zoom of objective, changes pixel size
- angle_deg :float # Angle of objective, changes rotation of image
- scan_params_dict :longblob # Other ScanM parameters less frequently used
+ scan_frequency=-1 :float32 # Scanning frequency in Hz
+ scan_period=-1 :float32 # Scanning duration per frame in seconds
+ line_duration=-1 :float32 # Line duration from OS_Parameters
+ real_pixel_duration=-1 :float32 # Duration of a pixel (un-cropped)
+ zoom :float32 # Zoom of objective, changes pixel size
+ angle_deg :float32 # Angle of objective, changes rotation of image
+ scan_params_dict : # Other ScanM parameters less frequently used
"""
return definition
@@ -353,8 +355,7 @@ def _add_entry(self, pres_key: dict, filepath: str, verboselvl: int = 0) -> None
for avg_entry in avg_entries:
self.StackAverages().insert1(avg_entry, allow_direct_insert=True)
- @staticmethod
- def _complete_keys(base_key: dict, rec: "ScanMRecording") -> tuple[dict, dict, list]:
+ def _complete_keys(self, base_key: dict, rec: "ScanMRecording") -> tuple[dict, dict, list]:
"""Build the three entry dicts needed to insert a presentation record.
Args:
@@ -368,7 +369,7 @@ def _complete_keys(base_key: dict, rec: "ScanMRecording") -> tuple[dict, dict, l
part-table rows (one per channel).
"""
pres_entry = deepcopy(base_key)
- pres_entry["pres_data_file"] = rec.filepath
+ pres_entry["pres_data_file"] = relative_store_path(rec.filepath, self._filepath_store)
pres_entry["trigger_valid"] = int(rec.trigger_valid)
pres_entry["triggertimes"] = rec.trigger_times.astype(np.float32)
@@ -419,9 +420,10 @@ def plot1(self, key: dict = None, gamma: float = 0.7) -> None:
key = get_primary_key(table=self, key=key)
npixartifact, scan_type = (self.field_table & key).fetch1('npixartifact', 'scan_type')
data_name, alt_name = (self.userinfo_table & key).fetch1('data_stack_name', 'alt_stack_name')
- main_ch_average = (self.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
+ main_ch_average = load_array((self.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
try:
- alt_ch_average = (self.StackAverages & key & f'ch_name="{alt_name}"').fetch1('ch_average')
+ alt_ch_average = load_array(
+ (self.StackAverages & key & dict(ch_name=alt_name)).fetch1('ch_average'))
except dj.DataJointError:
alt_ch_average = np.full_like(main_ch_average, np.nan)
plot_field(main_ch_average, alt_ch_average, scan_type=scan_type,
@@ -440,15 +442,16 @@ def plot1_triggers(self, key: dict = None) -> tuple:
"""
key = get_primary_key(table=self, key=key)
triggertimes, filepath = (self & key).fetch1('triggertimes', 'pres_data_file')
+ triggertimes = load_array(triggertimes)
setupid = (self.experiment_table().ExpInfo & key).fetch1("setupid")
isrepeated, ntrigger_rep = (self.stimulus_table & key).fetch1("isrepeated", "ntrigger_rep")
- rec = ScanMRecording(filepath=filepath, setup_id=setupid, date=key['date'],
- repeated_stim=isrepeated, ntrigger_rep=ntrigger_rep)
- rec.set_auto_trigger_threshold()
-
- trigger_trace = rec.ch_stacks[rec.trigger_ch_name].T.flatten()
+ with local_path(filepath, self._filepath_store) as filepath_local:
+ rec = ScanMRecording(filepath=filepath_local, setup_id=setupid, date=key['date'],
+ repeated_stim=isrepeated, ntrigger_rep=ntrigger_rep)
+ rec.set_auto_trigger_threshold()
+ trigger_trace = rec.ch_stacks[rec.trigger_ch_name].T.flatten()
fig, ax = plt.subplots(1, 1, figsize=(8, 4))
ax.set_title(f"{len(triggertimes)} triggers detected")
diff --git a/djimaging/tables/core/raw_data.py b/djimaging/tables/core/raw_data.py
index 6a611ce..d7306b8 100644
--- a/djimaging/tables/core/raw_data.py
+++ b/djimaging/tables/core/raw_data.py
@@ -12,14 +12,14 @@ class RawDataParamsTemplate(dj.Lookup):
def definition(self):
definition = """
-> self.userinfo_table
- raw_id: tinyint unsigned # unique param set id
+ raw_id: int32 # unique param set id
---
- from_raw_data: tinyint unsigned # Load raw smp data (1) or h5 data (0)
- compute_from_stack: tinyint unsigned # Compute traces from stack. Otherwise try to import Igor traces.
- include_artifacts: tinyint unsigned # automatically exclude all ROIs with artifacts?
- trace_precision: enum("line", "pixel") # Compute traces with either line precision or pixel precision?
- trigger_precision: enum("line", "pixel") # Compute triggers with either line precision or pixel precision?
- igor_roi_masks: enum("yes", "init", "no") # Either load or ignore existing ROI masks, e.g. from Igor
+ from_raw_data: bool # Load raw smp data (1) or h5 data (0)
+ compute_from_stack: bool # Compute traces from stack. Otherwise try to import Igor traces.
+ include_artifacts: bool # automatically exclude all ROIs with artifacts?
+ trace_precision: enum('line', 'pixel') # Compute traces with either line precision or pixel precision?
+ trigger_precision: enum('line', 'pixel') # Compute triggers with either line precision or pixel precision?
+ igor_roi_masks: enum('yes', 'init', 'no') # Either load or ignore existing ROI masks, e.g. from Igor
"""
return definition
@@ -64,7 +64,7 @@ def add_default(
Default is True.
"""
if experimenter_list is None:
- experimenter_list = self.userinfo_table.fetch('experimenter')
+ experimenter_list = self.userinfo_table.to_arrays('experimenter')
for experimenter in experimenter_list:
key = dict(
diff --git a/djimaging/tables/core/roi.py b/djimaging/tables/core/roi.py
index 7283715..f302b09 100644
--- a/djimaging/tables/core/roi.py
+++ b/djimaging/tables/core/roi.py
@@ -5,6 +5,7 @@
import datajoint as dj
import numpy as np
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_field
from djimaging.utils.scanm.roi_utils import extract_roi_ids
@@ -18,12 +19,12 @@ def definition(self):
definition = """
# ROI information
-> self.roi_mask_table
- roi_id :smallint # integer id of each ROI
+ roi_id :int32 # integer id of each ROI
---
- roi_size :int # number of pixels in ROI
- roi_size_um2 :float # size of ROI in micrometers squared
- roi_dia_um :float # diameter of ROI in micrometers, if it was a circle
- artifact_flag :tinyint unsigned # flag if roi contains light artifact (1) or not (0)
+ roi_size :int32 # number of pixels in ROI
+ roi_size_um2 :float32 # size of ROI in micrometers squared
+ roi_dia_um :float32 # diameter of ROI in micrometers, if it was a circle
+ artifact_flag :bool # flag if roi contains light artifact (1) or not (0)
"""
return definition
@@ -61,7 +62,7 @@ def make(self, key: dict) -> None:
be processed.
"""
# load roi_mask for insert roi for a specific experiment and field
- roi_mask = (self.roi_mask_table & key).fetch1("roi_mask")
+ roi_mask = load_array((self.roi_mask_table & key).fetch1("roi_mask"))
scan_type = (self.field_table & key).fetch1("scan_type")
pixel_size_um = (self.field_table & key).fetch1("pixel_size_um")
@@ -105,11 +106,13 @@ def plot1(self, key: dict = None, gamma: float = 0.7) -> None:
key = get_primary_key(table=self, key=key)
npixartifact, scan_type = (self.field_table & key).fetch1("npixartifact", "scan_type")
- roi_mask = (self.roi_mask_table & key).fetch1("roi_mask")
+ roi_mask = load_array((self.roi_mask_table & key).fetch1("roi_mask"))
data_name, alt_name = (self.userinfo_table() & key).fetch1('data_stack_name', 'alt_stack_name')
- main_ch_average = (self.field_table.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
- alt_ch_average = (self.field_table.StackAverages & key & f'ch_name="{alt_name}"').fetch1('ch_average')
+ main_ch_average = load_array(
+ (self.field_table.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
+ alt_ch_average = load_array(
+ (self.field_table.StackAverages & key & dict(ch_name=alt_name)).fetch1('ch_average'))
plot_field(main_ch_average, alt_ch_average, scan_type=scan_type,
roi_mask=roi_mask, roi_ch_average=main_ch_average,
diff --git a/djimaging/tables/core/roi_mask.py b/djimaging/tables/core/roi_mask.py
index 2a31d64..ae1372c 100644
--- a/djimaging/tables/core/roi_mask.py
+++ b/djimaging/tables/core/roi_mask.py
@@ -1,7 +1,6 @@
from __future__ import annotations
import os
-import pickle
import warnings
from abc import abstractmethod
@@ -13,9 +12,13 @@
from djimaging.autorois.roi_canvas import InteractiveRoiCanvas
from djimaging.utils.filesystem_utils import as_pre_filepath
+from djimaging.utils.dj_storage import file_store_path, load_array, local_file_path, relative_store_path
from djimaging.utils.dj_utils import get_primary_key, check_unique_one
-from djimaging.utils.mask_utils import to_roi_mask_file, sort_roi_mask_files, \
- load_preferred_roi_mask_igor, load_preferred_roi_mask_pickle, compare_roi_masks
+from djimaging.utils.mask_utils import (
+ to_roi_mask_file, sort_roi_mask_files,
+ load_preferred_roi_mask_igor, load_preferred_roi_mask_file, compare_roi_masks,
+ save_roi_mask_file, load_roi_mask_file, _ROI_FILE_FORMAT_HELP,
+)
from djimaging.utils.mask_format_utils import to_igor_format, to_python_format
from djimaging.utils.plot_utils import plot_field
@@ -23,6 +26,7 @@
class RoiMaskTemplate(dj.Manual):
database = ""
_max_shift = 5
+ _roi_file_format: str = 'numpy'
@property
def definition(self):
@@ -32,7 +36,7 @@ def definition(self):
-> self.raw_params_table
---
-> self.presentation_table
- roi_mask : blob # ROI mask for recording field
+ roi_mask : # ROI mask for recording field
"""
return definition
@@ -44,10 +48,10 @@ def definition(self):
-> master
-> self.presentation_table
---
- roi_mask : blob # ROI mask for presentation field
- as_field_mask : enum("same", "different", "shifted") # relationship to field mask
- shift_dx=0 : int # Shift in x
- shift_dy=0 : int # Shift in y
+ roi_mask : # ROI mask for presentation field
+ as_field_mask : enum('same', 'different', 'shifted') # relationship to field mask
+ shift_dx=0 : int32 # Shift in x
+ shift_dy=0 : int32 # Shift in y
"""
return definition
@@ -93,6 +97,17 @@ def highres_table(self) -> dj.Table:
"""Return the high-resolution stack table."""
pass
+ def _get_roi_file_format(self) -> str:
+ """Return the configured ROI file format, or raise a descriptive error if unset."""
+ if self._roi_file_format is None:
+ raise ValueError(
+ f"_roi_file_format is not set on {type(self).__name__}. "
+ f"{_ROI_FILE_FORMAT_HELP} "
+ f"Add `_roi_file_format = 'numpy'` (recommended) or "
+ f"`_roi_file_format = 'pickle'` for existing pickle masks as a class attribute."
+ )
+ return self._roi_file_format
+
@property
def key_source(self):
"""Return the key source for this manual table."""
@@ -108,7 +123,7 @@ def list_missing_field(self) -> list:
A list of primary key dicts for fields that are missing from this
ROI mask table.
"""
- missing_keys = (self.field_table.proj() & (self.presentation_table.proj() - self.proj())).fetch(as_dict=True)
+ missing_keys = (self.field_table.proj() & (self.presentation_table.proj() - self.proj())).to_dicts()
return missing_keys
def load_field_file_info_df(self, field_key: dict):
@@ -128,16 +143,16 @@ def load_field_file_info_df(self, field_key: dict):
def draw_roi_mask(
self,
- field_key: dict = None,
- pres_key: dict = None,
+ field_key: dict | None = None,
+ pres_key: dict | None = None,
canvas_width: int = 20,
autorois_models: str = 'default_rgc',
show_diagnostics: bool = True,
load_high_res: bool = True,
- max_shift: int = None,
+ max_shift: int | None = None,
roi_mask_dir: str = 'ROIs',
- old_prefix: str = None,
- new_prefix: str = None,
+ old_prefix: str | None = None,
+ new_prefix: str | None = None,
use_stim_onset: bool = True,
verbose: bool = True,
**kwargs,
@@ -174,13 +189,13 @@ def draw_roi_mask(
raise ValueError(f'canvas_width={canvas_width} must be in (0, 100)%')
if pres_key is not None:
- field_key = (self.field_table & pres_key).fetch1('KEY')
+ field_key = (self.field_table & pres_key).proj().fetch1()
elif field_key is None:
field_key = np.random.choice(self.list_missing_field())
if field_key is None:
raise ValueError('No field_key provided and no missing field found.')
- field_key = (self.field_table & field_key).fetch1('KEY')
+ field_key = (self.field_table & field_key).proj().fetch1()
from_raw_data = (self.raw_params_table & field_key).fetch1("from_raw_data")
@@ -191,7 +206,10 @@ def draw_roi_mask(
filepaths = []
for f in all_filepaths:
- pres_key_list = ((self.presentation_table & field_key) & dict(pres_data_file=f)).proj().fetch(as_dict=True)
+ store_path = relative_store_path(f, self.presentation_table._filepath_store)
+ pres_key_list = (
+ (self.presentation_table & field_key) & {"pres_data_file.path": store_path}
+ ).proj().to_dicts()
if len(pres_key_list) == 1:
pres_keys.append(pres_key_list[0])
filepaths.append(f)
@@ -205,7 +223,8 @@ def draw_roi_mask(
data_name, alt_name = (self.userinfo_table & field_key).fetch1('data_stack_name', 'alt_stack_name')
ch0_stacks, ch1_stacks, output_files = load_stack_data(
files=filepaths, data_name=data_name, alt_name=alt_name, from_raw_data=from_raw_data,
- roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix)
+ roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
+ file_format=self._get_roi_file_format())
# Some sanity checks
assert len(pres_names) == len(pres_keys)
@@ -215,7 +234,7 @@ def draw_roi_mask(
assert len(output_files) == len(pres_keys)
# Load pixel size and scan type
- n_artifact, pixel_size_um, scan_type = (self.presentation_table() & pres_keys).fetch(
+ n_artifact, pixel_size_um, scan_type = (self.presentation_table() & pres_keys).to_arrays(
'npixartifact', 'pixel_size_um', 'scan_type')
n_artifact = check_unique_one(n_artifact, name='n_artifact')
pixel_size_um = check_unique_one(pixel_size_um, name='pixel_size_um')
@@ -224,7 +243,7 @@ def draw_roi_mask(
if scan_type == 'xy':
pixel_size_d1_d2 = (pixel_size_um, pixel_size_um)
elif scan_type == 'xz':
- z_step_um = (self.presentation_table() & pres_keys).fetch('z_step_um')
+ z_step_um = (self.presentation_table() & pres_keys).to_arrays('z_step_um')
z_step_um = check_unique_one(z_step_um, name='z_step_um')
pixel_size_d1_d2 = (pixel_size_um, z_step_um)
else:
@@ -232,8 +251,8 @@ def draw_roi_mask(
# Reduce stacks to after stimulus onset?
if use_stim_onset:
- triggertimes = (self.presentation_table & pres_keys).fetch('triggertimes')
- scan_frequencies = (self.presentation_table.ScanInfo() & pres_keys).fetch('scan_frequency')
+ triggertimes = (self.presentation_table & pres_keys).to_arrays('triggertimes')
+ scan_frequencies = (self.presentation_table.ScanInfo() & pres_keys).to_arrays('scan_frequency')
stim_onset_idxs = [int(np.floor(tt[0] * fs)) if len(tt) > 0 else 0
for tt, fs in zip(triggertimes, scan_frequencies)]
@@ -319,7 +338,7 @@ def load_initial_roi_mask(self, field_key, igor_roi_masks=str, roi_mask_dir='ROI
"""
Load initial ROI mask for field.
First try to load from database.
- Second try pickle file, unless ROI masks should be exclusively loaded from Igor.
+ Second try filesystem file (numpy or pickle), unless ROI masks should be exclusively loaded from Igor.
Last try to load from Igor file, unless ROI masks should never be loaded from Igor.
"""
@@ -329,7 +348,7 @@ def load_initial_roi_mask(self, field_key, igor_roi_masks=str, roi_mask_dir='ROI
return roi_mask, 'database'
if igor_roi_masks != 'yes':
- roi_mask, src_file = self.load_field_roi_mask_pickle(
+ roi_mask, src_file = self.load_field_roi_mask_file(
field_key=field_key, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
verbose=verbose)
if roi_mask is not None:
@@ -355,18 +374,18 @@ def load_field_roi_mask_database(self, field_key: dict) -> np.ndarray | None:
Raises:
ValueError: If more than one ROI mask is found for the key.
"""
- database_roi_masks = (self & field_key).fetch("roi_mask")
+ database_roi_masks = (self & field_key).to_arrays("roi_mask")
if len(database_roi_masks) == 1:
database_roi_mask = database_roi_masks[0].copy()
elif len(database_roi_masks) == 0:
database_roi_mask = None
else:
- raise ValueError(f'Found multiple ROI masks for key=\n{field_key}\n{(self & field_key).fetch("KEY")}')
+ raise ValueError(f'Found multiple ROI masks for key=\n{field_key}\n{(self & field_key).keys()}')
return database_roi_mask
- def load_field_roi_mask_pickle(
+ def load_field_roi_mask_file(
self,
field_key: dict,
roi_mask_dir: str = 'ROIs',
@@ -374,11 +393,11 @@ def load_field_roi_mask_pickle(
new_prefix: str = None,
verbose: bool = True,
) -> tuple[np.ndarray | None, str | None]:
- """Load ROI mask from a pickle file on the filesystem.
+ """Load an ROI mask file in the configured NumPy or pickle format.
Args:
field_key: Primary key dict for the field.
- roi_mask_dir: Sub-directory name containing ROI mask pickle files.
+ roi_mask_dir: Sub-directory name containing ROI mask files.
old_prefix: Path prefix to replace when resolving file paths.
new_prefix: Replacement path prefix.
verbose: If True, print a message when a mask is loaded.
@@ -389,11 +408,15 @@ def load_field_roi_mask_pickle(
(or None if not found).
"""
mask_alias, highres_alias = (self.userinfo_table() & field_key).fetch1("mask_alias", "highres_alias")
- files = (self.presentation_table() & field_key).fetch("pres_data_file")
+ files = [
+ local_file_path(value, self.presentation_table._filepath_store)
+ for value in (self.presentation_table() & field_key).to_arrays("pres_data_file")
+ ]
- roi_mask, src_file = load_preferred_roi_mask_pickle(
+ roi_mask, src_file = load_preferred_roi_mask_file(
files, mask_alias=mask_alias, highres_alias=highres_alias,
- roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix)
+ roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
+ file_format=self._get_roi_file_format())
if roi_mask is not None and verbose:
print(f'Loaded ROI mask from file={src_file} for files=\n{files}\nfor mask_alias={mask_alias}')
@@ -417,7 +440,10 @@ def load_field_roi_mask_igor(
"""
mask_alias, highres_alias, raw_data_dir, pre_data_dir = (self.userinfo_table() & field_key).fetch1(
"mask_alias", "highres_alias", "raw_data_dir", "pre_data_dir")
- files = (self.presentation_table() & field_key).fetch("pres_data_file")
+ files = [
+ local_file_path(value, self.presentation_table._filepath_store)
+ for value in (self.presentation_table() & field_key).to_arrays("pres_data_file")
+ ]
files = [as_pre_filepath(f, raw_data_dir=raw_data_dir, pre_data_dir=pre_data_dir) for f in files]
@@ -427,11 +453,23 @@ def load_field_roi_mask_igor(
return roi_mask, src_file
- def rescan_filesystem(self, restrictions: dict = None, verboselvl: int = 0, suppress_errors: bool = False,
- only_new_fields: bool = True, roi_mask_dir='ROIs', old_prefix=None, new_prefix=None,
- max_shift=None, auto_fill_pres_keys: bool = False, add_primary_keys=None):
+ def rescan_filesystem(
+ self,
+ restrictions: dict | None = None,
+ source = 'infer', # 'infer' | 'igor' | 'autorois'
+ verboselvl: int = 0,
+ suppress_errors: bool = False,
+ only_new_fields: bool = True,
+ roi_mask_dir='ROIs',
+ old_prefix=None,
+ new_prefix=None,
+ max_shift=None,
+ auto_fill_pres_keys: bool = False,
+ add_primary_keys=None
+ ):
"""Scan filesystem for new ROI masks and add them to the database.
:param restrictions: Restrictions for field_table
+ :param source: Source of ROI masks to scan for. Can be 'infer' (to automatically infer from parameters), 'igor' (to only scan for Igor files) or 'autorois' (to only scan for AutoROIs files).
:param verboselvl: Verbosity level
:param suppress_errors: Suppress errors
:param only_new_fields: Only scan for new fields
@@ -446,14 +484,22 @@ def rescan_filesystem(self, restrictions: dict = None, verboselvl: int = 0, supp
restrictions = dict()
if only_new_fields:
- restrictions = (self.key_source - self) & restrictions
+ keys_to_scan = ((self.key_source - self) & restrictions).proj().to_dicts()
+ else:
+ keys_to_scan = (self.key_source & restrictions).proj().to_dicts()
+
+ if verboselvl > 0:
+ print(f'Scanning {len(keys_to_scan)} keys for new ROI masks...')
err_list = []
- for key in (self.key_source & restrictions):
+ for key in keys_to_scan:
+ if verboselvl > 2:
+ print('#' * 60)
+ print('Scanning key:', key)
try:
self._add_field_roi_masks(
- key, auto_fill_pres_keys=auto_fill_pres_keys,
+ key, source=source, auto_fill_pres_keys=auto_fill_pres_keys,
roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
max_shift=max_shift, add_primary_keys=add_primary_keys, verboselvl=verboselvl)
except Exception as e:
@@ -465,33 +511,147 @@ def rescan_filesystem(self, restrictions: dict = None, verboselvl: int = 0, supp
return err_list
+ def check_filesystem_consistency(
+ self,
+ restrictions: dict | None = None,
+ source = 'infer', # 'infer' | 'igor' | 'autorois'
+ roi_mask_dir: str = 'ROIs',
+ old_prefix: str | None = None,
+ new_prefix: str | None = None,
+ verbose: bool = True,
+ ) -> list:
+ """Check whether database ROI masks are consistent with the filesystem.
+
+ Iterates over every presentation already stored in RoiMaskPresentation
+ and compares it against the corresponding file on disk. Returns a list
+ of problem records so the caller can inspect or correct them.
+
+ Args:
+ restrictions: Optional DataJoint restriction applied to
+ RoiMaskPresentation before iterating (e.g. a field key dict).
+ source: Source of ROI masks to check for. Can be 'infer' (to automatically infer from parameters), 'igor' (to only check Igor files) or 'autorois' (to only check AutoROIs files).
+ roi_mask_dir: Sub-directory name containing ROI mask files.
+ old_prefix: Path prefix to replace when resolving file paths.
+ new_prefix: Replacement path prefix.
+ verbose: If True, print progress and a summary at the end.
+
+ Returns:
+ A list of dicts, one per problematic presentation, each with:
+
+ - ``'key'``: the presentation primary key dict
+ - ``'issue'``: one of
+
+ * ``'mismatch'`` — file exists but its content differs from DB
+ * ``'missing_on_filesystem'`` — DB has a mask but the pickle
+ file is gone
+ * ``'igor_missing'`` — DB has a mask but the H5 file contains
+ no ROI mask
+
+ - ``'filesystem_file'``: expected file path (pickle or H5)
+
+ Example::
+
+ problems = roi_mask_table.check_filesystem_consistency(
+ roi_mask_dir='ROIs')
+ for p in problems:
+ print(p['issue'], p['filesystem_file'])
+ print(p['key'])
+ """
+ if restrictions is None:
+ restrictions = dict()
+
+ problems = []
+ pres_keys = (self.RoiMaskPresentation & restrictions).proj().to_dicts()
+
+ if verbose:
+ print(f'Checking filesystem consistency for {len(pres_keys)} presentation(s)...')
+
+ for i, key in enumerate(pres_keys):
+ if source == 'infer':
+ igor_roi_masks = (self.raw_params_table & key).fetch1('igor_roi_masks')
+ elif source == 'igor':
+ igor_roi_masks = 'yes'
+ elif source == 'autorois':
+ igor_roi_masks = 'no'
+
+ input_file = local_file_path(
+ (self.presentation_table & key).fetch1('pres_data_file'),
+ self.presentation_table._filepath_store)
+ roimask_file = to_roi_mask_file(
+ input_file, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
+ file_format=self._get_roi_file_format())
+
+ if verbose:
+ print(f' [{i + 1}/{len(pres_keys)}] Checking key={key}')
+
+ if igor_roi_masks == 'yes':
+ filesystem_file = input_file
+ filesystem_roi_mask = read_h5_utils.load_roi_mask(
+ filepath=input_file, ignore_not_found=True)
+ if filesystem_roi_mask is None:
+ if verbose:
+ print(f' -> PROBLEM: igor_missing ({filesystem_file})')
+ problems.append(
+ {'key': key, 'issue': 'igor_missing', 'filesystem_file': filesystem_file})
+ continue
+ else:
+ filesystem_file = roimask_file
+ if not os.path.isfile(roimask_file):
+ if verbose:
+ print(f' -> PROBLEM: missing_on_filesystem ({filesystem_file})')
+ problems.append(
+ {'key': key, 'issue': 'missing_on_filesystem', 'filesystem_file': filesystem_file})
+ continue
+ filesystem_roi_mask = load_roi_mask_file(roimask_file).astype(np.int32)
+ filesystem_roi_mask = to_igor_format(filesystem_roi_mask)
+
+ database_roi_mask = load_array((self.RoiMaskPresentation & key).fetch1('roi_mask'))
+ if not np.all(filesystem_roi_mask == database_roi_mask):
+ if verbose:
+ print(f' -> PROBLEM: mismatch ({filesystem_file})')
+ problems.append(
+ {'key': key, 'issue': 'mismatch', 'filesystem_file': filesystem_file})
+ else:
+ if verbose:
+ print(f' -> OK')
+
+ if verbose:
+ if problems:
+ print(f'Done. Found {len(problems)} problem(s) out of {len(pres_keys)} presentation(s).')
+ else:
+ print(f'Done. All {len(pres_keys)} presentation(s) are consistent.')
+
+ return problems
+
def _add_field_roi_masks(
self,
field_key: dict,
+ source: str = 'infer',
auto_fill_pres_keys: bool = False,
- add_primary_keys: dict = None,
+ add_primary_keys: dict | None = None,
roi_mask_dir: str = 'ROIs',
- old_prefix: str = None,
- new_prefix: str = None,
- max_shift: int = None,
+ old_prefix: str | None = None,
+ new_prefix: str | None = None,
+ max_shift: int | None = None,
verboselvl: int = 0,
) -> None:
"""Find and insert ROI mask entries for all presentations of a field.
Args:
field_key: Primary key dict for the field.
+ source: Source of the ROI masks ('infer', 'igor', or 'autorois').
auto_fill_pres_keys: If True, presentations without a mask file
are filled with the main field mask (zero shift).
add_primary_keys: Optional extra primary key columns to inject into
every inserted row.
- roi_mask_dir: Sub-directory name containing ROI mask pickle files.
+ roi_mask_dir: Sub-directory name containing ROI mask files.
old_prefix: Path prefix to replace when resolving file paths.
new_prefix: Replacement path prefix.
max_shift: Maximum allowed pixel shift between presentations.
Defaults to the table-level ``_max_shift`` value.
verboselvl: Verbosity level controlling diagnostic output.
"""
- pres_keys = (self.presentation_table & field_key).fetch('KEY')
+ pres_keys = (self.presentation_table & field_key).keys()
if add_primary_keys:
pres_keys = [{**pk, **add_primary_keys} for pk in pres_keys]
@@ -500,39 +660,43 @@ def _add_field_roi_masks(
if verboselvl > 2:
print('\nfield_key:', field_key, '\npres_keys:', pres_keys)
- roi_masks = [self._load_presentation_roi_mask(key, roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix)
- for key in pres_keys]
+ # Filter to new presentation keys BEFORE loading masks, so that existing
+ # entries are not needlessly read from disk and their filesystem-vs-database
+ # consistency is not checked here (which would cause rescan to fail if any
+ # pre-existing mask file was edited outside the database).
+ new_pres_keys = [pk for pk in pres_keys
+ if len(self.RoiMaskPresentation & pk) == 0]
+
+ if len(new_pres_keys) == 0:
+ if verboselvl > 1:
+ print('Nothing new to add for field:', field_key)
+ if verboselvl > 2:
+ print('pres_keys:', [k for k in pres_keys])
+ return
- data_pairs = zip(pres_keys, roi_masks)
+ roi_masks = [self._load_presentation_roi_mask(
+ key, roi_mask_dir, source=source, old_prefix=old_prefix, new_prefix=new_prefix)
+ for key in new_pres_keys]
if not auto_fill_pres_keys:
# Filter out keys without ROI mask
- data_pairs = [(pres_key, roi_mask) for pres_key, roi_mask in data_pairs
+ data_pairs = [(pres_key, roi_mask) for pres_key, roi_mask in zip(new_pres_keys, roi_masks)
if roi_mask is not None]
else:
- data_pairs = list(data_pairs)
+ data_pairs = list(zip(new_pres_keys, roi_masks))
if len(data_pairs) == 0:
if verboselvl > 1:
print('No ROI masks found for field:', field_key)
- if verboselvl > 2:
- print('pres_keys:', [k for k in pres_keys])
- return
-
- # Filter out keys that are already present
- data_pairs = [(pres_key, roi_mask) for pres_key, roi_mask in data_pairs
- if len(self.RoiMaskPresentation & pres_key) == 0]
-
- if len(data_pairs) == 0:
- if verboselvl > 1:
- print('Nothing new to add for field:', field_key)
return
if len(self & field_key) == 0:
# Find preferred file that should be used as main key.
mask_alias, highres_alias = (self.userinfo_table & field_key).fetch1("mask_alias", "highres_alias")
keys_masks_files = [
- (pres_key, roi_mask, (self.presentation_table & pres_key).fetch1('pres_data_file'))
+ (pres_key, roi_mask, local_file_path(
+ (self.presentation_table & pres_key).fetch1('pres_data_file'),
+ self.presentation_table._filepath_store))
for pres_key, roi_mask in data_pairs
if roi_mask is not None]
@@ -550,8 +714,8 @@ def _add_field_roi_masks(
main_pres_key, main_roi_mask = keys_masks_files[sort_idxs[0]][:2]
else:
- main_pres_key = (self.RoiMaskPresentation().proj() & (self & field_key)).fetch1('KEY')
- main_roi_mask = (self.RoiMaskPresentation & main_pres_key).fetch1('roi_mask')
+ main_pres_key = (self.RoiMaskPresentation().proj() & (self & field_key)).fetch1()
+ main_roi_mask = load_array((self.RoiMaskPresentation & main_pres_key).fetch1('roi_mask'))
roi_mask_pres_keys = []
@@ -577,7 +741,6 @@ def _add_field_roi_masks(
for roi_mask_pres_key in roi_mask_pres_keys:
if add_primary_keys is not None:
- main_key = {**main_key, **add_primary_keys}
roi_mask_pres_key = {**roi_mask_pres_key, **add_primary_keys}
self.RoiMaskPresentation().insert1(roi_mask_pres_key, skip_duplicates=True)
@@ -586,14 +749,15 @@ def _load_presentation_roi_mask(
self,
key: dict,
roi_mask_dir: str = 'ROIs',
- old_prefix: str = None,
- new_prefix: str = None,
+ source: str = 'infer',
+ old_prefix: str | None = None,
+ new_prefix: str | None = None,
) -> np.ndarray | None:
"""Load the ROI mask for a single presentation, reconciling filesystem and database state.
Args:
key: Primary key dict for the presentation.
- roi_mask_dir: Sub-directory name containing ROI mask pickle files.
+ roi_mask_dir: Sub-directory name containing ROI mask files.
old_prefix: Path prefix to replace when resolving file paths.
new_prefix: Replacement path prefix.
@@ -603,33 +767,50 @@ def _load_presentation_roi_mask(
Raises:
ValueError: If the filesystem mask differs from the database mask.
"""
- igor_roi_masks, from_raw_data = (self.raw_params_table & key).fetch1('igor_roi_masks', 'from_raw_data')
- input_file = (self.presentation_table & key).fetch1("pres_data_file")
-
+ input_file = local_file_path(
+ (self.presentation_table & key).fetch1("pres_data_file"),
+ self.presentation_table._filepath_store)
+
+ if source == 'infer':
+ igor_roi_masks, from_raw_data = (self.raw_params_table & key).fetch1('igor_roi_masks', 'from_raw_data')
+ elif source == 'igor':
+ igor_roi_masks, from_raw_data = 'yes', False
+ elif source == 'autorois':
+ igor_roi_masks, from_raw_data = 'no', True
+
+ file_format = self._get_roi_file_format()
roimask_file = to_roi_mask_file(
- input_file, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix)
+ input_file, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
+ file_format=file_format)
if igor_roi_masks == 'yes':
assert not from_raw_data, 'Inconsistent parameters'
filesystem_roi_mask = read_h5_utils.load_roi_mask(filepath=input_file, ignore_not_found=True)
else:
if os.path.isfile(roimask_file):
- with open(roimask_file, 'rb') as f:
- filesystem_roi_mask = pickle.load(f).copy().astype(np.int32)
+ filesystem_roi_mask = load_roi_mask_file(roimask_file).astype(np.int32)
filesystem_roi_mask = to_igor_format(filesystem_roi_mask)
else:
filesystem_roi_mask = None
if len((self.RoiMaskPresentation & key).proj()) > 0:
- database_roi_mask = (self.RoiMaskPresentation & key).fetch1('roi_mask')
+ database_roi_mask = load_array((self.RoiMaskPresentation & key).fetch1('roi_mask'))
if filesystem_roi_mask is None:
- warnings.warn(
- f'ROI mask for key=\n{key}\nhas been deleted on the filesystem but not in the database.\n'
- f'Saving ROI masks to file now: {roimask_file}'
- )
- with open(roimask_file, 'wb') as f:
- pickle.dump(to_python_format(database_roi_mask), f)
+ if igor_roi_masks == 'yes':
+ # H5 files have no write-back API; writing a pickle would be silently
+ # ignored on the next scan (which still reads from H5), causing an
+ # infinite repeated warning.
+ warnings.warn(
+ f'ROI mask for key=\n{key}\nwas not found in the H5 file but exists in the database. '
+ f'Cannot write back to H5 format; database entry retained.'
+ )
+ else:
+ warnings.warn(
+ f'ROI mask for key=\n{key}\nhas been deleted on the filesystem but not in the database.\n'
+ f'Saving ROI masks to file now: {roimask_file}'
+ )
+ save_roi_mask_file(roimask_file, to_python_format(database_roi_mask), file_format)
elif not np.all(filesystem_roi_mask == database_roi_mask):
raise ValueError(f'ROI mask for key=\n{key}\nhas been changed on filesystem but not in database.')
@@ -638,7 +819,7 @@ def _load_presentation_roi_mask(
return filesystem_roi_mask
- def plot1(self, key: dict = None, gamma: float = 0.5) -> None:
+ def plot1(self, key: dict | None = None, gamma: float = 0.5) -> None:
"""Plot the stack average with the ROI mask overlay for one presentation.
Args:
@@ -649,18 +830,19 @@ def plot1(self, key: dict = None, gamma: float = 0.5) -> None:
key = get_primary_key(table=self.proj() * self.presentation_table.proj(), key=key)
npixartifact, scan_type = (self.field_table & key).fetch1('npixartifact', 'scan_type')
data_name, alt_name = (self.userinfo_table & key).fetch1('data_stack_name', 'alt_stack_name')
- main_ch_average = (self.presentation_table.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
+ main_ch_average = load_array(
+ (self.presentation_table.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
try:
- alt_ch_average = (self.presentation_table.StackAverages & key & f'ch_name="{alt_name}"').fetch1(
- 'ch_average')
+ alt_ch_average = load_array(
+ (self.presentation_table.StackAverages & key & dict(ch_name=alt_name)).fetch1('ch_average'))
except dj.DataJointError:
alt_ch_average = np.full_like(main_ch_average, np.nan)
- roi_mask = (self.RoiMaskPresentation & key).fetch1('roi_mask')
+ roi_mask = load_array((self.RoiMaskPresentation & key).fetch1('roi_mask'))
plot_field(main_ch_average, alt_ch_average, scan_type=scan_type,
roi_mask=roi_mask, title=key, npixartifact=npixartifact, gamma=gamma)
- def load_high_res_bg_dict(self, key: dict, ch_names: list, verbose: bool = True) -> dict:
+ def load_high_res_bg_dict(self, key: dict | None, ch_names: list, verbose: bool = True) -> dict:
"""Load high-resolution channel averages as a background dict for the GUI.
Args:
@@ -675,7 +857,7 @@ def load_high_res_bg_dict(self, key: dict, ch_names: list, verbose: bool = True)
"""
try:
ch_names, ch_averages, *conds = (
- self.highres_table().StackAverages & key & [f"ch_name='{cn}'" for cn in ch_names]).fetch(
+ self.highres_table().StackAverages & key & [f"ch_name='{cn}'" for cn in ch_names]).to_arrays(
'ch_name', 'ch_average', *self.highres_table().new_primary_keys)
bg_dict = dict()
@@ -699,8 +881,9 @@ def load_stack_data(
alt_name: str,
from_raw_data: bool,
roi_mask_dir: str = 'ROIs',
- old_prefix: str = None,
- new_prefix: str = None,
+ old_prefix: str | None = None,
+ new_prefix: str | None = None,
+ file_format: str | None = None,
) -> tuple[list, list, list]:
"""Load image stacks for a list of presentation files.
@@ -712,6 +895,8 @@ def load_stack_data(
roi_mask_dir: Sub-directory name for ROI mask output files.
old_prefix: Path prefix to replace in output file paths.
new_prefix: Replacement path prefix.
+ file_format: ``'numpy'`` or ``'pickle'``; controls the suffix of the
+ output file paths passed to the GUI for saving.
Returns:
A 3-tuple of (ch0_stacks, ch1_stacks, output_files) where each is a
@@ -726,7 +911,8 @@ def load_stack_data(
ch0_stacks.append(ch_stacks[data_name])
ch1_stacks.append(ch_stacks[alt_name])
output_files.append(to_roi_mask_file(
- data_file, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix))
+ data_file, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
+ file_format=file_format))
return ch0_stacks, ch1_stacks, output_files
@@ -757,8 +943,10 @@ def _add_autorois_unet(autorois_models: dict) -> None:
"""
try:
from djimaging.autorois.unet import UNet
- config_path = "/gpfs01/euler/data/Resources/AutoROIs/models/UNET_v0.1.0/sd_images.yaml"
- checkpoint_path = "/gpfs01/euler/data/Resources/AutoROIs/models/UNET_v0.1.0/dropout_and_aug_regul.ckpt"
+ config_path = file_store_path(
+ "Resources/AutoROIs/models/UNET_v0.1.0/sd_images.yaml", "reference")
+ checkpoint_path = file_store_path(
+ "Resources/AutoROIs/models/UNET_v0.1.0/dropout_and_aug_regul.ckpt", "reference")
unet_model = UNet.from_checkpoint(config_path, checkpoint_path)
autorois_models['UNet'] = unet_model
diff --git a/djimaging/tables/core/snippets.py b/djimaging/tables/core/snippets.py
index 2c52816..d4673c4 100644
--- a/djimaging/tables/core/snippets.py
+++ b/djimaging/tables/core/snippets.py
@@ -7,12 +7,30 @@
import numpy as np
from matplotlib import pyplot as plt
-from djimaging.tables.core.stimulus import reformat_numerical_trial_info
from djimaging.utils import plot_utils
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.snippet_utils import split_trace_by_reps, split_trace_by_group_reps, compute_repeat_correlation
+def fetch_snippets_and_times(
+ snippets_table: dj.Table | type[dj.Table],
+ key: dict,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+ """Fetch repeated snippets, absolute sample times, and trigger times as arrays.
+
+ This handles SnippetsTemplate rows with shape (time, repetitions), including
+ arrays stored externally. GroupSnippetsTemplate rows contain dictionaries
+ and must be handled separately.
+ """
+ snippets_t0, snippets_dt, snippets, triggertimes = (snippets_table & key).fetch1(
+ 'snippets_t0', 'snippets_dt', 'snippets', 'triggertimes_snippets')
+ snippets, snippets_t0, triggertimes = (
+ load_array(snippets), load_array(snippets_t0), load_array(triggertimes))
+ snippets_times = (np.arange(snippets.shape[0]) * snippets_dt)[:, None] + snippets_t0
+ return snippets, snippets_times, triggertimes
+
+
def get_aligned_snippets_times(
snippets_times: np.ndarray,
raise_error: bool = True,
@@ -52,18 +70,6 @@ def get_aligned_snippets_times(
class SnippetsTemplate(dj.Computed):
database = ""
_pad_trace = False # If True, chose snippet times always contain the trigger times
- _dt_base_line_dict = None # dict of baseline time for each stimulus with stimulus based baseline correction
-
- """
- Examples for _dt_base_line_dict:
- This is deprecated and should be replaced by the snippet_base_dt in the stimulus table.
-
- Baden 16 / Franke 17:
- _dt_base_line_dict = {
- 'gChirp': 8*0.128,
- 'movingbar': 5*0.128,
- }
- """
@property
def definition(self):
@@ -71,11 +77,11 @@ def definition(self):
# Snippets created from slicing traces using the triggertimes.
-> self.preprocesstraces_table
---
- snippets :longblob # array of snippets (time x repetitions)
- snippets_t0 :blob # array of snippet start times (repetitions, )
- snippets_dt :float
- triggertimes_snippets :longblob # snippeted triggertimes (ntrigger_rep x repetitions)
- droppedlastrep_flag :tinyint unsigned # Was the last repetition incomplete and therefore dropped?
+ snippets : # array of snippets (time x repetitions)
+ snippets_t0 : # array of snippet start times (repetitions, )
+ snippets_dt :float32
+ triggertimes_snippets : # snippeted triggertimes (ntrigger_rep x repetitions)
+ droppedlastrep_flag :bool # Was the last repetition incomplete and therefore dropped?
"""
return definition
@@ -123,9 +129,10 @@ def make(self, key: dict) -> None:
"""
stim_name, stim_dict, ntrigger_rep = (self.stimulus_table() & key).fetch1(
'stim_name', 'stim_dict', 'ntrigger_rep')
- triggertimes = (self.presentation_table() & key).fetch1('triggertimes')
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
pp_trace_t0, pp_trace_dt, pp_trace = (self.preprocesstraces_table() & key).fetch1(
'pp_trace_t0', 'pp_trace_dt', 'pp_trace')
+ pp_trace = load_array(pp_trace)
delay = stim_dict.get('trigger_delay', 0.) if stim_dict is not None else 0
@@ -150,40 +157,9 @@ def make(self, key: dict) -> None:
))
def get_snippet_base_dt(self, stim_name: str) -> float | None:
- """Return the baseline duration for snippet baseline correction.
-
- Checks first in the stimulus table's ``snippet_base_dt`` column, then
- falls back to the class-level ``_dt_base_line_dict`` lookup.
-
- Args:
- stim_name: Stimulus name used to look up the baseline duration.
-
- Returns:
- Baseline window duration in seconds, or None if no baseline
- correction should be applied.
-
- Raises:
- ValueError: If both sources provide different non-None values.
- """
- try:
- dt_baseline = (self.stimulus_table & dict(stim_name=stim_name)).fetch1('snippet_base_dt')
- if not np.isfinite(dt_baseline):
- dt_baseline = None
- except dj.DataJointError:
- dt_baseline = None
-
- dt_baseline_alt = None if self._dt_base_line_dict is None else self._dt_base_line_dict.get(stim_name, None)
-
- if dt_baseline is not None and dt_baseline_alt is not None:
- if dt_baseline != dt_baseline_alt:
- raise ValueError(
- f"dt_baseline[Stimulus]={dt_baseline} and dt_baseline[Snippets]={dt_baseline_alt} are not equal. "
- f"Please set only one of them, ideally in the stimulus table."
- )
- elif dt_baseline is None and dt_baseline_alt is not None:
- dt_baseline = dt_baseline_alt
-
- return dt_baseline
+ """Return the stimulus's baseline duration, or None when unset."""
+ dt_baseline = (self.stimulus_table() & dict(stim_name=stim_name)).fetch1('snippet_base_dt')
+ return dt_baseline if dt_baseline is not None and np.isfinite(dt_baseline) else None
def plot1(self, key: dict = None, xlim: tuple = None, xlim_aligned: tuple = None) -> None:
"""Plot the raw, repetition-aligned, and averaged snippets for one entry.
@@ -195,11 +171,7 @@ def plot1(self, key: dict = None, xlim: tuple = None, xlim_aligned: tuple = None
xlim_aligned: x-axis limits for the aligned snippet panels.
"""
key = get_primary_key(table=self, key=key)
- snippets_t0, snippets_dt, snippets, triggertimes_snippets = (self & key).fetch1(
- "snippets_t0", "snippets_dt", "snippets", "triggertimes_snippets")
-
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
+ snippets, snippets_times, triggertimes_snippets = fetch_snippets_and_times(self, key)
fig, axs = plt.subplots(3, 1, figsize=(10, 6))
@@ -266,11 +238,11 @@ def definition(self):
# Snippets created from slicing traces using the triggertimes.
-> self.preprocesstraces_table
---
- snippets :longblob # dict of array of snippets (group: time [x repetitions])
- snippets_t0 :blob # dict of array of snippet start times (group: repetitions)
- snippets_dt :float
- triggertimes_snippets :longblob # dict of array of triggertimes (group: time [x repetitions])
- droppedlastrep_flag :tinyint unsigned # Was the last repetition incomplete and therefore dropped?
+ snippets : # dict of array of snippets (group: time [x repetitions])
+ snippets_t0 : # dict of array of snippet start times (group: repetitions)
+ snippets_dt :float32
+ triggertimes_snippets : # dict of array of triggertimes (group: time [x repetitions])
+ droppedlastrep_flag :bool # Was the last repetition incomplete and therefore dropped?
"""
return definition
@@ -317,15 +289,13 @@ def make(self, key: dict, allow_incomplete: bool = True) -> None:
allow_incomplete: If True, allow incomplete last repetitions.
"""
trial_info, stim_dict = (self.stimulus_table() & key).fetch1('trial_info', 'stim_dict')
- triggertimes = (self.presentation_table() & key).fetch1('triggertimes')
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
pp_trace_t0, pp_trace_dt, pp_trace = (self.preprocesstraces_table() & key).fetch1(
'pp_trace_t0', 'pp_trace_dt', 'pp_trace')
+ pp_trace = load_array(pp_trace)
pp_trace_times = np.arange(len(pp_trace)) * pp_trace_dt + pp_trace_t0
- if not isinstance(trial_info[0], dict):
- trial_info = reformat_numerical_trial_info(trial_info)
-
delay = stim_dict.get('trigger_delay', 0.) if stim_dict is not None else 0
snippets, snippets_times, triggertimes_snippets, droppedlastrep_flag = split_trace_by_group_reps(
diff --git a/djimaging/tables/core/stim_logs.py b/djimaging/tables/core/stim_logs.py
index 0ad664c..f3a75ac 100644
--- a/djimaging/tables/core/stim_logs.py
+++ b/djimaging/tables/core/stim_logs.py
@@ -3,6 +3,47 @@
from djimaging.utils.qdspy.read_logs import parse_stimulus_log
+def _parse_scan_start_time(scan_params_dict: dict):
+ """Extract recording start time from a ScanM scan_params_dict.
+
+ Returns a datetime.time on success, or None if the dict lacks the required
+ keys or none of the known formats match.
+ """
+ from datetime import datetime as _dt
+
+ def _decode(v):
+ return v.decode('utf-8') if hasattr(v, 'decode') else str(v)
+
+ date_str = None
+ for k in ('datestamp_d_m_y', 'datestamp_y_m_d', 'date', 'datestamp'):
+ if scan_params_dict.get(k) is not None:
+ date_str = _decode(scan_params_dict[k])
+ break
+
+ time_str = None
+ for k in ('timestamp_h_m_s_ms', 'timestamp', 'time'):
+ if scan_params_dict.get(k) is not None:
+ time_str = _decode(scan_params_dict[k])
+ break
+
+ if date_str is None or time_str is None:
+ return None
+
+ for fmt in (
+ '%Y-%m-%d %H-%M-%S-%f',
+ '%Y-%m-%d %H:%M:%S.%f',
+ '%Y-%m-%d %H:%M:%S',
+ '%d-%m-%Y %H-%M-%S-%f',
+ '%d-%m-%Y %H:%M:%S.%f',
+ '%d-%m-%Y %H:%M:%S',
+ ):
+ try:
+ return _dt.strptime(f'{date_str} {time_str}', fmt).time()
+ except ValueError:
+ pass
+ return None
+
+
class QdsPyLogFileTemplate(dj.Manual):
database = ""
@@ -10,7 +51,7 @@ class QdsPyLogFileTemplate(dj.Manual):
def definition(self):
definition = """
-> self.exp_table
- log_idx : tinyint
+ log_idx : int32
---
log_path : varchar(1024)
"""
@@ -28,7 +69,7 @@ def add1(self, key, log_path: str, **kwargs):
key: Key identifying the experiment.
log_path: Path to the log file.
"""
- log_idxs, log_paths = (self & key).fetch('log_idx', 'log_path')
+ log_idxs, log_paths = (self & key).to_arrays('log_idx', 'log_path')
if len(log_idxs) == 0:
log_idx = 0
else:
@@ -53,10 +94,10 @@ def definition(self):
definition = """
-> self.log_file_table
---
- n_lines_total : int
- n_lines_data : int
- n_lines_err : int
- n_err : int
+ n_lines_total : int32
+ n_lines_data : int32
+ n_lines_err : int32
+ n_err : int32
"""
return definition
@@ -64,28 +105,27 @@ class StimLog(dj.Part):
@property
def definition(self):
definition = """
- # ROI Mask
-> master
- stim_idx : int # stimulus index
+ stim_idx : int32 # stimulus index
---
stim_file_name : varchar(255)
stim_path : varchar(1024)
stim_md5 : char(32)
-
- t_abs_s : float # absolute time (s)
- t_since_last_s : float # time since previous stim (s)
-
+
+ t_abs_s : float32 # absolute time (s)
+ t_since_last_s : float32 # time since previous stim (s)
+
t_start : time
t_end : time
-
aborted : bool
- t_dur_s : float
- t_dur_s_calc : float
-
- n_dropped_frames : int
-
- params : blob
- other_info : blob # additional info with new keys
+ t_dur_s : float32
+ t_dur_s_calc : float32
+
+ n_dropped_frames : int32
+
+ params :
+ sequence_used = null : int32
+ other_info : # additional info with new keys
"""
return definition
@@ -110,6 +150,13 @@ def make(self, key, root_dir=None, verbose: bool = False):
'n_err': stats.get('nErr', -1),
})
+ _KNOWN_STIM_KEYS = {
+ 'index', 'stimFileName', 'stimPath', 'stimMD5',
+ 't_abs_s', 't_since_last_s', 't_start', 't_end',
+ 'aborted', 't_dur_s', 't_dur_s_calc', 'nDroppedFrames',
+ 'params', 'sequenceUsed',
+ }
+
stim_rows = []
for stim in stims:
stim_row = key.copy()
@@ -127,13 +174,217 @@ def make(self, key, root_dir=None, verbose: bool = False):
't_dur_s_calc': stim.get('t_dur_s_calc', -1),
'n_dropped_frames': stim.get('nDroppedFrames', -1),
'params': stim.get('params', dict()),
- 'other_info': {k: v for k, v in stim.items() if k not in [
- 'stimFileName', 'stimPath', 'stimMD5', 't_abs_s', 't_since_last_s',
- 't_start', 't_end', 'aborted', 't_dur_s', 't_dur_s_calc',
- 'nDroppedFrames', 'params'
- ]}
+ 'sequence_used': stim.get('sequenceUsed', None),
+ 'other_info': {k: v for k, v in stim.items() if k not in _KNOWN_STIM_KEYS},
})
stim_rows.append(stim_row)
self.insert1(row)
self.StimLog().insert(stim_rows)
+
+
+class PresentationLogTemplate(dj.Computed):
+ database = ""
+
+ @property
+ def definition(self):
+ definition = """
+ -> self.presentation_table
+ ---
+ log_idx : int8
+ stim_idx : int32
+ match_method : varchar(32)
+ t_start : time
+ t_end : time
+ aborted : bool
+ """
+ return definition
+
+ @property
+ def exp_table(self):
+ """Override to specify the experiment table (used in make() to scope log queries)."""
+ raise NotImplementedError("Subclasses must implement the exp_table property.")
+
+ @property
+ def presentation_table(self):
+ """Override to specify the presentation table."""
+ raise NotImplementedError("Subclasses must implement the presentation_table property.")
+
+ @property
+ def log_table(self):
+ """Override to specify the log table (a QdsPyLogTemplate.StimLog instance)."""
+ raise NotImplementedError("Subclasses must implement the log_table property.")
+
+ @property
+ def stimulus_table(self):
+ """Override to specify the stimulus table (must expose stim_hash and stim_dict)."""
+ raise NotImplementedError("Subclasses must implement the stimulus_table property.")
+
+ @property
+ def scan_params_table(self):
+ """Override to enable time-based matching.
+
+ Should return a table (e.g. Presentation.ScanInfo) that has a
+ scan_params_dict longblob field fetchable by the presentation key.
+ Return None (default) to skip time matching and use ordinal matching only.
+ """
+ return None
+
+ def make(self, key, verbose: int = 0, skip_if_no_log: bool = False):
+ if verbose >= 1:
+ print(f"[PresentationLog] Processing key: {key}")
+
+ exp_key = (self.exp_table & key).fetch1('KEY')
+
+ stim_hash, stim_dict = (self.stimulus_table & key).fetch1('stim_hash', 'stim_dict')
+ hashes = {stim_hash} | set(stim_dict.get('stim_hash_list', []))
+ hashes.discard('')
+ names = set(stim_dict.get('stim_name_list', []))
+ if 'stim_name' in key:
+ names.add(key['stim_name'])
+
+ if verbose >= 2:
+ print(f" stim_hash={stim_hash!r}")
+ print(f" candidate hashes : {hashes}")
+ print(f" candidate names : {names}")
+
+ def _fetch_log_rows(include_aborted: bool):
+ q = self.log_table & exp_key
+ if not include_aborted:
+ q = q & 'aborted = 0'
+ return q.to_dicts(order_by=('log_idx', 'stim_idx'))
+
+ def _filter_rows(rows):
+ names_lower = {n.lower() for n in names}
+ by_md5 = [r for r in rows if r['stim_md5'] in hashes]
+ if by_md5:
+ return by_md5, 'md5'
+ by_name = [
+ r for r in rows
+ if any(r['stim_file_name'].lower().startswith(n) for n in names_lower)
+ ]
+ return by_name, 'filename'
+
+ log_rows = _fetch_log_rows(include_aborted=False)
+
+ if verbose >= 2:
+ print(f" non-aborted log rows for experiment: {len(log_rows)}")
+ if verbose >= 3:
+ for r in log_rows:
+ print(f" log_idx={r['log_idx']} stim_idx={r['stim_idx']} "
+ f"name={r['stim_file_name']!r} md5={r['stim_md5']!r}")
+
+ matches, method = _filter_rows(log_rows)
+
+ if not matches:
+ if verbose >= 2:
+ print(" no match in non-aborted rows — retrying with aborted entries included")
+ log_rows = _fetch_log_rows(include_aborted=True)
+ if not log_rows:
+ if skip_if_no_log:
+ if verbose >= 1:
+ print(" no log entries for this experiment — skipping")
+ return
+ raise ValueError(f"No log entries at all for experiment key={exp_key}.")
+ if verbose >= 3:
+ aborted_rows = [r for r in log_rows if r['aborted']]
+ for r in aborted_rows:
+ print(f" [aborted] log_idx={r['log_idx']} stim_idx={r['stim_idx']} "
+ f"name={r['stim_file_name']!r} md5={r['stim_md5']!r}")
+ matches, method = _filter_rows(log_rows)
+
+ if verbose >= 2:
+ print(f" match_method={method!r}, {len(matches)} candidate(s) after filtering "
+ f"({sum(r['aborted'] for r in matches)} aborted)")
+
+ if not matches:
+ raise ValueError(
+ f"No log entry found for key={key}. "
+ f"Checked hashes={hashes}, names={names}."
+ )
+
+ # --- selection: time-based (preferred) or ordinal (fallback) ---
+
+ matched = None
+
+ if len(matches) == 1:
+ matched = matches[0]
+
+ scan_params_table = self.scan_params_table
+ if matched is None and scan_params_table is not None:
+ try:
+ scan_params_dict = (scan_params_table & key).fetch1('scan_params_dict')
+ rec_time = _parse_scan_start_time(scan_params_dict)
+ if rec_time is not None:
+ rec_s = (rec_time.hour * 3600 + rec_time.minute * 60
+ + rec_time.second + rec_time.microsecond / 1e6)
+
+ def _delta_s(row):
+ # DataJoint returns MySQL TIME columns as timedelta
+ t = row['t_start']
+ log_s = t.total_seconds() if hasattr(t, 'total_seconds') else (
+ t.hour * 3600 + t.minute * 60 + t.second)
+ return abs(rec_s - log_s)
+ matched = min(matches, key=_delta_s)
+ method = method + '+time'
+ if verbose >= 2:
+ print(f" time-matching: recording start={rec_time}, "
+ f"best match t_start={matched['t_start']} "
+ f"(delta={_delta_s(matched):.0f}s)")
+ if verbose >= 3:
+ for r in matches:
+ marker = " <-- selected" if r is matched else ""
+ print(f" stim_idx={r['stim_idx']} t_start={r['t_start']} "
+ f"delta={_delta_s(r):.0f}s{marker}")
+ else:
+ if verbose >= 2:
+ print(" time-matching: could not parse time from scan_params_dict")
+ except Exception as e:
+ if verbose >= 2:
+ print(f" time-matching failed ({e}) — falling back to ordinal matching")
+
+ if matched is None:
+ stim_pk = self.stimulus_table.primary_key
+ stim_filter = {k: key[k] for k in stim_pk if k in key}
+ all_pres_keys = sorted(
+ (self.presentation_table & exp_key & stim_filter).proj().to_dicts(),
+ key=lambda k: tuple(v for _, v in sorted(k.items()))
+ )
+ pres_idx = next(
+ (i for i, pk in enumerate(all_pres_keys)
+ if all(key.get(k) == pk.get(k) for k in pk)),
+ None
+ )
+
+ if verbose >= 2:
+ print(f" ordinal matching: {len(all_pres_keys)} sibling presentation(s), "
+ f"this ordinal={pres_idx}")
+ if verbose >= 3:
+ for i, pk in enumerate(all_pres_keys):
+ marker = " <-- this" if i == pres_idx else ""
+ print(f" [{i}] {pk}{marker}")
+
+ if pres_idx is None:
+ raise ValueError(f"Could not locate key={key} among sibling presentations.")
+ if len(matches) <= pres_idx:
+ raise ValueError(
+ f"Only {len(matches)} log entr(ies) found for "
+ f"hashes={hashes} / names={names}, but presentation ordinal is {pres_idx}."
+ )
+ matched = matches[pres_idx]
+
+ if verbose >= 1:
+ aborted_flag = " [ABORTED]" if matched['aborted'] else ""
+ print(f" -> matched log_idx={matched['log_idx']} stim_idx={matched['stim_idx']} "
+ f"method={method!r} t_start={matched['t_start']} t_end={matched['t_end']}"
+ f"{aborted_flag}")
+
+ self.insert1({
+ **key,
+ 'log_idx' : matched['log_idx'],
+ 'stim_idx' : matched['stim_idx'],
+ 'match_method': method,
+ 't_start' : matched['t_start'],
+ 't_end' : matched['t_end'],
+ 'aborted' : matched['aborted'],
+ })
diff --git a/djimaging/tables/core/stimulus.py b/djimaging/tables/core/stimulus.py
index 9a2c919..058fa11 100644
--- a/djimaging/tables/core/stimulus.py
+++ b/djimaging/tables/core/stimulus.py
@@ -1,101 +1,65 @@
from __future__ import annotations
import warnings
+from collections.abc import Iterable
+from pathlib import Path
import datajoint as dj
import numpy as np
-try:
- from collections.abc import Iterable
-except ImportError:
- from collections import Iterable
-
-
-def reformat_numerical_trial_info(trial_info: list) -> list:
- """Change old list format to new list[dict] format.
-
- Args:
- trial_info: List of numerical trial identifiers (e.g. direction angles).
-
- Returns:
- A list of dicts with keys 'name' and 'ntrigger' (always 1) derived
- from the input list.
- """
- return [dict(name=trial_info_i, ntrigger=1) for i, trial_info_i in enumerate(trial_info)]
+from djimaging.utils.dj_storage import relative_store_path
def check_trial_info(trial_info: list, ntrigger_rep: int) -> list:
- """Validate trial info and normalise to the list[dict] format if necessary.
-
- Args:
- trial_info: Trial information as either a list of dicts (each with
- 'name' and 'ntrigger' keys) or a legacy list of numerical values.
- ntrigger_rep: Expected total number of triggers per repetition. Must
- equal the sum of 'ntrigger' values across all trial_info entries.
-
- Returns:
- Trial info in the canonical list[dict] format.
-
- Raises:
- TypeError: If trial_info is not a list or array.
- AssertionError: If an unknown key is present in a trial_info entry or
- if a trigger count is not an integer.
- ValueError: If the trigger count sum does not match ntrigger_rep (only
- raised when ntrigger_rep > 1).
- """
- if not isinstance(trial_info, (list, np.ndarray)):
- raise TypeError('trial_info must either be a list or an array')
-
- if not isinstance(trial_info[0], dict):
- trial_info = reformat_numerical_trial_info(trial_info)
-
- for trial_info_i in trial_info:
- for k, v in trial_info_i.items():
- assert k in ['name', 'ntrigger', 'ntrigger_split'], f'Unknown key in trial_info k={k}'
-
- if k in ['ntrigger', 'ntrigger_split']:
- assert int(v) == v, f'Value for k={k} but be an integer but is v={v}'
-
- ntrigger_ti = sum([trial_info_i["ntrigger"] for trial_info_i in trial_info])
- if not (ntrigger_ti == ntrigger_rep):
- msg = f'Number of triggers in trial_info={ntrigger_ti} must match ntrigger_rep={ntrigger_rep}.'
- if ntrigger_rep > 1:
- raise ValueError(msg)
- else:
- # Raise only warnings for previous work-around solution
- warnings.warn(msg)
+ """Validate canonical trial dictionaries and their trigger counts.
+ ``ntrigger_rep == 1`` denotes one snippet per trial, as used for moving
+ bars. Otherwise the trial sequence must contain ``ntrigger_rep`` triggers.
+ """
+ if not isinstance(trial_info, (list, np.ndarray)) or len(trial_info) == 0:
+ raise TypeError('trial_info must be a nonempty list of dictionaries')
+
+ for trial in trial_info:
+ if not isinstance(trial, dict):
+ raise TypeError("Each trial must be a dictionary with 'name' and 'ntrigger' keys")
+ if not {'name', 'ntrigger'} <= trial.keys():
+ raise ValueError("Each trial must define 'name' and 'ntrigger'")
+ for key, value in trial.items():
+ if key not in ('name', 'ntrigger', 'ntrigger_split'):
+ raise ValueError(f'Unknown trial_info key: {key}')
+ if key in ('ntrigger', 'ntrigger_split') and (int(value) != value or value < 1):
+ raise ValueError(f'{key} must be a positive integer, got {value}')
+
+ ntrigger_total = sum(trial['ntrigger'] for trial in trial_info)
+ if ntrigger_rep != 1 and ntrigger_total != ntrigger_rep:
+ raise ValueError(f'Number of triggers in trial_info={ntrigger_total} must match ntrigger_rep={ntrigger_rep}.')
return trial_info
class StimulusTemplate(dj.Manual):
database = ""
- _incl_snippet_base_dt = False # Optional for compatibility with previous versions
+ _filepath_store = "reference"
@property
def definition(self):
- definition = """
+ definition = f"""
# Light stimuli
stim_name :varchar(32) # Unique string identifier
---
alias :varchar(999) # Strings (_ seperator) to identify this stimulus, not case sensitive!
stim_family="" :varchar(191) # To group stimuli (e.g. gChirp and lChirp) for downstream processing
- framerate=0 :float # framerate in Hz
- isrepeated=0 :tinyint unsigned # Is the stimulus repeated? Used for snippets
- ntrigger_rep=0 :mediumint unsigned # Number of triggers (per repetition)
- stim_path="" :varchar(191) # Path to hdf5 file containing numerical array and info about stim
+ framerate=0 :float32 # framerate in Hz
+ isrepeated=0 :bool # Is the stimulus repeated? Used for snippets
+ ntrigger_rep=0 :int32 # Number of triggers (per repetition)
+ stim_path=NULL : # optional source stimulus file
commit_id="" :varchar(191) # Commit id corresponding to stimulus entry in GitHub repo
stim_hash="" :varchar(191) # QDSpy hash
- trial_info=NULL :longblob # trial information, e.g. directions of moving bar
- stim_trace=NULL :longblob # array of stimulus if available
- stim_dict=NULL :longblob # stimulus information dictionary, contains e.g. spatial extent
+ snippet_base_dt=NULL :float32 # Time used for snippet baseline estimation
+ trial_info=NULL : # trial information, e.g. directions of moving bar
+ stim_trace=NULL : # array of stimulus if available
+ stim_dict=NULL : # stimulus information dictionary, contains e.g. spatial extent
"""
- if self._incl_snippet_base_dt:
- definition += """
- snippet_base_dt=NULL : float # Time used for snippet baseline estimation
- """
-
return definition
def check_alias(self, alias: str, stim_name: str) -> None:
@@ -110,7 +74,7 @@ def check_alias(self, alias: str, stim_name: str) -> None:
AssertionError: If any individual alias token already exists in the
table for a different stimulus.
"""
- existing_aliases = (self - [dict(stim_name=stim_name)]).fetch('alias') # Skip duplicate comparison
+ existing_aliases = (self - [dict(stim_name=stim_name)]).to_arrays('alias') # Skip duplicate comparison
for existing_alias in existing_aliases:
for existing_alias_i in existing_alias.split('_'):
assert existing_alias_i not in alias.split('_'), \
@@ -118,7 +82,7 @@ def check_alias(self, alias: str, stim_name: str) -> None:
def add_stimulus(self, stim_name: str, alias: str, stim_family: str = "", framerate: float = 0,
isrepeated: bool = 0, ntrigger_rep: int = 0, snippet_base_dt: float = None,
- stim_path: str = "", commit_id: str = "",
+ stim_path: str | Path | None = None, commit_id: str = "",
trial_info: Iterable = None, stim_trace: np.ndarray = None, stim_dict: dict = None,
skip_duplicates: bool = False, unique_alias: bool = True) -> None:
"""
@@ -152,7 +116,7 @@ def add_stimulus(self, stim_name: str, alias: str, stim_family: str = "", framer
if trial_info is not None:
# noinspection PyTypeChecker
- check_trial_info(trial_info=trial_info, ntrigger_rep=ntrigger_rep)
+ trial_info = check_trial_info(trial_info=trial_info, ntrigger_rep=ntrigger_rep)
key = {
"stim_name": stim_name,
@@ -161,42 +125,19 @@ def add_stimulus(self, stim_name: str, alias: str, stim_family: str = "", framer
"ntrigger_rep": int(np.round(ntrigger_rep)),
"isrepeated": int(np.round(isrepeated)),
"framerate": framerate,
- "stim_path": stim_path,
+ "stim_path": (
+ relative_store_path(stim_path, self._filepath_store)
+ if stim_path not in (None, "") else None
+ ),
"commit_id": commit_id,
+ "snippet_base_dt": snippet_base_dt,
"trial_info": trial_info,
"stim_trace": stim_trace,
"stim_dict": stim_dict,
}
- if self._incl_snippet_base_dt:
- key["snippet_base_dt"] = snippet_base_dt
- elif snippet_base_dt is not None:
- raise ValueError(
- 'Snippet base dt is not supported for this table. '
- 'Set `_incl_snippet_base_dt` to True in the table definition or remove the parameter.'
- )
-
self.insert1(key, skip_duplicates=skip_duplicates)
- def update_trial_info_format(self, restriction: dict = None) -> None:
- """Update all trial info entries to the list[dict] format.
-
- Note: This modifies existing rows in-place and breaks compatibility
- with code that expects the old numerical list format.
-
- Args:
- restriction: Optional restriction dict applied before fetching
- keys. Defaults to no restriction (all entries).
- """
- if restriction is None:
- restriction = dict()
-
- for key in (self & restriction).proj().fetch(as_dict=True):
- trial_info, ntrigger_rep = (self & key).fetch1('trial_info', 'ntrigger_rep')
- if trial_info is not None:
- trial_info = check_trial_info(trial_info=trial_info, ntrigger_rep=ntrigger_rep)
- self.update1(dict(**key, trial_info=trial_info))
-
def add_nostim(self, alias: str = "nostim_none", skip_duplicates: bool = False) -> None:
"""Add a placeholder 'no stimulus' entry to the table.
@@ -223,8 +164,9 @@ def add_noise(
isrepeated: bool = False,
snippet_base_dt: float = None,
alias: str = None,
- ntrigger_per_frame: int = 1,
+ nframes_per_trigger: int = 1,
stim_trace: np.ndarray = None,
+ stim_path: str | Path | None = None,
pix_n_x: int = None,
pix_n_y: int = None,
pix_scale_x_um: float = None,
@@ -246,9 +188,10 @@ def add_noise(
isrepeated: Whether the stimulus is repeated. Default is False.
snippet_base_dt: Baseline window duration for snippet correction.
alias: Custom alias string; auto-generated from pixel scale if None.
- ntrigger_per_frame: Number of triggers sent per stimulus frame.
+ nframes_per_trigger: Number of stimulus frames per trigger.
Default is 1.
stim_trace: Optional numerical stimulus trace array.
+ stim_path: Optional source stimulus file in the configured reference store.
pix_n_x: Number of stimulus pixels in x.
pix_n_y: Number of stimulus pixels in y.
pix_scale_x_um: Pixel scale in x (micrometers).
@@ -267,7 +210,7 @@ def add_noise(
alias = f"dn_noise_dn{pix_scale_x_um}m_noise{pix_scale_x_um}m"
stim_dict = {
- "ntrigger_per_frame": ntrigger_per_frame,
+ "nframes_per_trigger": nframes_per_trigger,
}
if n_colors is not None:
@@ -309,6 +252,7 @@ def add_noise(
skip_duplicates=skip_duplicates,
unique_alias=True,
stim_trace=stim_trace,
+ stim_path=stim_path,
stim_dict=stim_dict,
)
@@ -317,11 +261,12 @@ def add_chirp(
stim_name: str = "chirp",
stim_family: str = 'chirp',
spatialextent: float = None,
- framerate: float = 1 / 60.,
+ framerate: float = 60.,
ntrigger_rep: int = 2,
isrepeated: bool = True,
snippet_base_dt: float = None,
stim_trace: np.ndarray = None,
+ stim_path: str | Path | None = None,
alias: str = None,
skip_duplicates: bool = False,
) -> None:
@@ -331,11 +276,12 @@ def add_chirp(
stim_name: Unique string identifier. Default is 'chirp'.
stim_family: Stimulus family label. Default is 'chirp'.
spatialextent: Spatial extent of the stimulus in micrometers.
- framerate: Stimulus frame rate in Hz. Default is 1/60.
+ framerate: Stimulus frame rate in Hz. Default is 60 [Hz].
ntrigger_rep: Number of triggers per repetition. Default is 2.
isrepeated: Whether the stimulus is repeated. Default is True.
snippet_base_dt: Baseline window duration for snippet correction.
stim_trace: Optional numerical stimulus trace array.
+ stim_path: Optional source stimulus file in the configured reference store.
alias: Custom alias string; defaults to a standard chirp alias if
None.
skip_duplicates: If True, silently skip duplicate entries.
@@ -360,6 +306,7 @@ def add_chirp(
skip_duplicates=skip_duplicates,
unique_alias=True,
stim_trace=stim_trace,
+ stim_path=stim_path,
stim_dict=stim_dict,
)
@@ -370,13 +317,14 @@ def add_movingbar(
ntrigger_rep: int = 1,
isrepeated: bool = True,
trial_info: list = None,
- framerate: float = 1 / 60.,
+ framerate: float = 60.,
snippet_base_dt: float = None,
bardx: float = None,
bardy: float = None,
velumsec: float = None,
tmovedurs: float = None,
stim_dict_other: dict = None,
+ stim_path: str | Path | None = None,
alias: str = None,
skip_duplicates: bool = False,
) -> None:
@@ -390,7 +338,7 @@ def add_movingbar(
trial_info: List of trial dicts (each with 'name' and 'ntrigger').
If None, defaults to the standard 8-direction protocol
[0, 180, 45, 225, 90, 270, 135, 315].
- framerate: Stimulus frame rate in Hz. Default is 1/60.
+ framerate: Stimulus frame rate in Hz. Default is 60 Hz.
snippet_base_dt: Baseline window duration for snippet correction.
bardx: Bar width in micrometers.
bardy: Bar height in micrometers.
@@ -398,6 +346,7 @@ def add_movingbar(
tmovedurs: Movement duration in seconds.
stim_dict_other: Additional key-value pairs to merge into the
stimulus dict.
+ stim_path: Optional source stimulus file in the configured reference store.
alias: Custom alias string; defaults to a standard moving-bar alias
if None.
skip_duplicates: If True, silently skip duplicate entries.
@@ -405,7 +354,7 @@ def add_movingbar(
"""
if trial_info is None:
- trial_info = np.array([0, 180, 45, 225, 90, 270, 135, 315])
+ trial_info = [dict(name=direction, ntrigger=1) for direction in (0, 180, 45, 225, 90, 270, 135, 315)]
# ===============================================================================
# Version "a574d90497d97ff9b93381f6b2eb207d"
# Example users:
@@ -469,6 +418,7 @@ def add_movingbar(
snippet_base_dt=snippet_base_dt,
framerate=framerate,
trial_info=trial_info,
+ stim_path=stim_path,
stim_dict=stim_dict,
skip_duplicates=skip_duplicates,
unique_alias=True,
diff --git a/djimaging/tables/core/traces.py b/djimaging/tables/core/traces.py
index 14d0f67..d10aa0b 100644
--- a/djimaging/tables/core/traces.py
+++ b/djimaging/tables/core/traces.py
@@ -7,12 +7,13 @@
import numpy as np
from matplotlib import pyplot as plt
-from djimaging.tables.core.preprocesstraces import plot_left_right_clipping
+from djimaging.tables.core.preprocesstraces import gui_clip_trace
from djimaging.utils.scanm.traces_and_triggers_utils import roi2trace_from_stack, check_valid_triggers_rel_to_tracetime
from djimaging.utils.scanm.read_h5_utils import load_roi2trace
from djimaging.utils import plot_utils, math_utils, trace_utils
+from djimaging.utils.dj_storage import load_array, local_path
from djimaging.utils.dj_utils import get_primary_key
-from djimaging.utils.plot_utils import plot_trace_and_trigger, prep_long_title
+from djimaging.utils.plot_utils import plot_trace_and_trigger
class TracesTemplate(dj.Computed):
@@ -36,11 +37,11 @@ def definition(self):
definition += """
---
- trace :longblob # array of raw trace
- trace_t0 :float # numerical array of trace times
- trace_dt :float # time between frames
- trace_valid :tinyint unsigned # Are values in trace correct (1) or not (0)?
- trigger_valid :tinyint unsigned # Are triggertimes inside trace_times (1) or not (0)?
+ trace : # array of raw trace
+ trace_t0 :float32 # numerical array of trace times
+ trace_dt :float32 # time between frames
+ trace_valid :bool # Are values in trace correct (1) or not (0)?
+ trigger_valid :bool # Are triggertimes inside trace_times (1) or not (0)?
"""
return definition
@@ -109,15 +110,16 @@ def make(self, key: dict, verboselvl: int = 0) -> None:
if from_raw_data and not compute_from_stack:
raise ValueError("from_raw_data=True only supported for compute_from_stack=True")
- filepath = (self.presentation_table & key).fetch1("pres_data_file")
- triggertimes = (self.presentation_table & key).fetch1("triggertimes")
- roi_ids = (self.roi_table & key).fetch("roi_id")
+ filepath_ref = (self.presentation_table & key).fetch1("pres_data_file")
+ triggertimes = load_array((self.presentation_table & key).fetch1("triggertimes"))
+ roi_ids = (self.roi_table & key).to_arrays("roi_id")
- if compute_from_stack:
- roi2trace, frame_dt = self._compute_roi2trace_from_stack(
- key, filepath, roi_ids, trace_precision, from_raw_data, verboselvl=verboselvl)
- else:
- roi2trace, frame_dt = load_roi2trace(filepath, roi_ids)
+ with local_path(filepath_ref, self.presentation_table._filepath_store) as filepath:
+ if compute_from_stack:
+ roi2trace, frame_dt = self._compute_roi2trace_from_stack(
+ key, filepath, roi_ids, trace_precision, from_raw_data, verboselvl=verboselvl)
+ else:
+ roi2trace, frame_dt = load_roi2trace(filepath, roi_ids)
for roi_id, roi_data in roi2trace.items():
if not include_artifacts and roi_data.get('incl_artifact', False):
@@ -168,6 +170,7 @@ def _compute_roi2trace_from_stack(
"""
data_stack_name = (self.userinfo_table & key).fetch1("data_stack_name")
roi_mask, as_field_mask = (self.roi_mask_table.RoiMaskPresentation & key).fetch1("roi_mask", "as_field_mask")
+ roi_mask = load_array(roi_mask)
n_artifact = (self.presentation_table & key).fetch1("npixartifact")
if as_field_mask == 'different':
@@ -176,6 +179,7 @@ def _compute_roi2trace_from_stack(
if self._include_motion_correction and (self.motion_detection_table is not None):
shifts_x, shifts_y = (self.motion_detection_table & key).fetch1('shifts_x', 'shifts_y')
+ shifts_x, shifts_y = load_array(shifts_x), load_array(shifts_y)
fs = (self.presentation_table.ScanInfo & key).fetch1('scan_frequency')
roi2trace, frame_dt = roi2trace_from_stack(
filepath=filepath, roi_ids=roi_ids, roi_mask=roi_mask,
@@ -207,31 +211,14 @@ def gui_clip_trace(self, key: dict) -> None:
"""
trace, trace_t0, trace_dt, valid_trace, valid_trigger = (self & key).fetch1(
'trace', 'trace_t0', 'trace_dt', 'trace_valid', 'trigger_valid')
- trace_t = np.arange(trace.size) * trace_dt + trace_t0
-
- import ipywidgets as widgets
-
- w_left = widgets.IntSlider(0, min=0, max=trace.size - 1, step=1,
- layout=widgets.Layout(width='800px'))
- w_right = widgets.IntSlider(trace.size - 1, min=0, max=trace.size - 1, step=1,
- layout=widgets.Layout(width='800px'))
- w_save = widgets.Checkbox(False)
-
- title = 'Not saved\n' + prep_long_title(key)
-
- @widgets.interact(left=w_left, right=w_right, save=w_save)
- def plot_fit(left=0, right=trace.size - 1, save=False):
- nonlocal title
+ trace = load_array(trace)
- plot_left_right_clipping(trace, trace_t, left, right, title)
+ def save_clip(i0: int, i1: int) -> None:
+ self.add_or_update(
+ key, trace=trace[i0:i1], trace_t0=trace_t0 + i0 * trace_dt, trace_dt=trace_dt,
+ valid_trace=valid_trace, valid_trigger=valid_trigger)
- if save:
- i0, i1 = (right, left + 1) if right < left else (left, right + 1)
- self.add_or_update(
- key, trace=trace[i0:i1], trace_t0=trace_t0 + left * trace_dt, trace_dt=trace_dt,
- valid_trace=valid_trace, valid_trigger=valid_trigger)
- title = f'SAVED: left={left}, right={right}\n{prep_long_title(key)}'
- w_save.value = False
+ gui_clip_trace(trace, trace_t0, trace_dt, key, save_clip)
def add_or_update(
self,
@@ -267,7 +254,8 @@ def plot1(self, key: dict = None, xlim: tuple = None, ylim: tuple = None) -> Non
"""
key = get_primary_key(table=self, key=key)
trace_t0, trace_dt, trace = (self & key).fetch1("trace_t0", "trace_dt", "trace")
- triggertimes = (self.presentation_table() & key).fetch1("triggertimes")
+ trace = load_array(trace)
+ triggertimes = load_array((self.presentation_table() & key).fetch1("triggertimes"))
trace_times = np.arange(len(trace)) * trace_dt + trace_t0
ax = plot_trace_and_trigger(
@@ -285,7 +273,7 @@ def plot(self, restriction: dict = None, sort: bool = True) -> None:
if restriction is None:
restriction = dict()
- traces = (self & restriction).fetch("trace")
+ traces = (self & restriction).to_arrays("trace")
traces = math_utils.padded_vstack(traces, cval=np.nan)
n = traces.shape[0]
diff --git a/djimaging/tables/core/userinfo.py b/djimaging/tables/core/userinfo.py
index 362d6d9..42218f8 100644
--- a/djimaging/tables/core/userinfo.py
+++ b/djimaging/tables/core/userinfo.py
@@ -18,15 +18,15 @@ def definition(self):
# Info for decoding file names
experimenter :varchar(16) # name of the experimenter
---
- data_dir :varchar(191) # path to header file, used for computed tables
- field_loc :tinyint # string location for field
- stimulus_loc :tinyint # string location for stimulus
- animal_loc=NULL :tinyint # string location for number of animal (e.g. M1)
- datatype_loc=NULL :tinyint # string location for datatype (e.g. SMP)
- region_loc=NULL :tinyint # string location for region (e.g. LR or RR)
- cond1_loc=NULL :tinyint # string location for condition 1 (e.g. pharmacological)
- cond2_loc=NULL :tinyint # string location for condition 2 (e.g. pharmacological)
- cond3_loc=NULL :tinyint # string location for condition 3 (e.g. pharmacological)
+ data_dir :varchar(191) # acquisition-tree root directory used for file discovery
+ field_loc :int32 # string location for field
+ stimulus_loc :int32 # string location for stimulus
+ animal_loc=NULL :int32 # string location for number of animal (e.g. M1)
+ datatype_loc=NULL :int32 # string location for datatype (e.g. SMP)
+ region_loc=NULL :int32 # string location for region (e.g. LR or RR)
+ cond1_loc=NULL :int32 # string location for condition 1 (e.g. pharmacological)
+ cond2_loc=NULL :int32 # string location for condition 2 (e.g. pharmacological)
+ cond3_loc=NULL :int32 # string location for condition 3 (e.g. pharmacological)
opticdisk_alias='od_opticdisk_opticdisc' :varchar(191) # alias(es) for optic disk (separated by _)
outline_alias='outline_edge' :varchar(191) # alias(es) for retinal outline / edge (separated by _)
highres_alias='hq_hr_highresolution_512' :varchar(191) # alias(es) for high resolution stack
@@ -77,7 +77,7 @@ def upload_user(self, userdict: dict, verbose: int = 1) -> None:
if 'alias' in k:
userdict[k] = v.lower()
- if userdict["experimenter"] not in self.fetch("experimenter"):
+ if userdict["experimenter"] not in self.to_arrays("experimenter"):
self.insert([userdict])
else:
entry = (self & dict(experimenter=userdict["experimenter"])).fetch1()
diff --git a/djimaging/tables/ipl/ipl.py b/djimaging/tables/ipl/ipl.py
index 25ccdcb..c2282a2 100644
--- a/djimaging/tables/ipl/ipl.py
+++ b/djimaging/tables/ipl/ipl.py
@@ -24,6 +24,7 @@ class RoiIplDepth(ipl.RoiIplDepthTemplate):
import numpy as np
from djimaging.utils.dj_utils import get_primary_key
+from djimaging.utils.dj_storage import load_array
from matplotlib import pyplot as plt
from djimaging.utils.plot_utils import plot_roi_mask_boundaries, prep_long_title
@@ -41,9 +42,9 @@ def definition(self):
# Use XZ widget notebook to determine values.
-> self.field_or_pres_table
---
- left :tinyint # pixel index where gcl/ipl border intersects on left of image (with GCL up)
- right :tinyint # pixel index where gcl/ipl border intersects on right side of image
- thick :tinyint # pixel width of the ipl
+ left :int32 # pixel index where gcl/ipl border intersects on left of image (with GCL up)
+ right :int32 # pixel index where gcl/ipl border intersects on right side of image
+ thick :int32 # pixel width of the ipl
"""
return definition
@@ -71,7 +72,7 @@ def list_missing_keys(self) -> list:
List of dicts, each representing one key in the key source that has not
yet been entered in this table.
"""
- missing_keys = (self.key_source - self.proj()).fetch(as_dict=True)
+ missing_keys = (self.key_source - self.proj()).to_dicts()
return missing_keys
def fetch1_and_norm_ch_avgs(self, key: dict, q_clip0: float = 0.0,
@@ -88,10 +89,12 @@ def fetch1_and_norm_ch_avgs(self, key: dict, q_clip0: float = 0.0,
"""
data_stack_name, alt_stack_name = (self.userinfo_table & key).fetch1('data_stack_name', 'alt_stack_name')
- ch0_avg = (self.field_or_pres_table.StackAverages & key & dict(ch_name=data_stack_name)).fetch1('ch_average')
+ ch0_avg = load_array(
+ (self.field_or_pres_table.StackAverages & key & dict(ch_name=data_stack_name)).fetch1('ch_average'))
ch0_avg = normalize_soft_zero_one(ch0_avg, dq=q_clip0)
- ch1_avg = (self.field_or_pres_table.StackAverages & key & dict(ch_name=alt_stack_name)).fetch1('ch_average')
+ ch1_avg = load_array(
+ (self.field_or_pres_table.StackAverages & key & dict(ch_name=alt_stack_name)).fetch1('ch_average'))
ch1_avg = normalize_soft_zero_one(ch1_avg, dq=q_clip1)
return ch0_avg, ch1_avg
@@ -164,7 +167,7 @@ def definition(self):
-> self.ipl_border_table
-> self.roi_table
---
- ipl_depth : float # Depth in the IPL relative to the GCL (=0) and INL (=1)
+ ipl_depth : float32 # Depth in the IPL relative to the GCL (=0) and INL (=1)
"""
return definition
@@ -198,8 +201,8 @@ def make(self, key: dict) -> None:
"""
left, right, thick = (self.ipl_border_table & key).fetch1('left', 'right', 'thick')
- roi_mask = (self.roimask_table & key).fetch1('roi_mask')
- roi_ids = (self.roi_table & key).fetch("roi_id")
+ roi_mask = load_array((self.roimask_table & key).fetch1('roi_mask'))
+ roi_ids = (self.roi_table & key).to_arrays("roi_id")
roi_centers = get_roi_centers(roi_mask, roi_ids)
@@ -226,7 +229,7 @@ def plot_field(self, key: dict = None, figsize: tuple = (8, 5),
key = get_primary_key(self.key_source & self)
left, right, thick = (self.ipl_border_table & key).fetch1('left', 'right', 'thick')
- roi_mask = (self.roimask_table & key).fetch1('roi_mask')
+ roi_mask = load_array((self.roimask_table & key).fetch1('roi_mask'))
ch0_avg, ch1_avg = self.ipl_border_table().fetch1_and_norm_ch_avgs(key, q_clip0=q_clip0, q_clip1=q_clip1)
@@ -235,7 +238,7 @@ def plot_field(self, key: dict = None, figsize: tuple = (8, 5),
fig, axs = plot_field_and_fit(
left=left, right=right, thick=thick, ch0_avg=ch0_avg, ch1_avg=ch1_avg, figsize=figsize, title=str(key))
- roi_idxs, ipl_depths = (self & key).fetch('roi_id', 'ipl_depth')
+ roi_idxs, ipl_depths = (self & key).to_arrays('roi_id', 'ipl_depth')
# get colormap for ipl depths
cmap = plt.cm.get_cmap('viridis')
diff --git a/djimaging/tables/ipl/ipl_bin.py b/djimaging/tables/ipl/ipl_bin.py
index 64a1a6d..6abaffb 100644
--- a/djimaging/tables/ipl/ipl_bin.py
+++ b/djimaging/tables/ipl/ipl_bin.py
@@ -19,6 +19,8 @@ class RoiIplBin(ipl.RoiIplBinTemplate):
import datajoint as dj
import numpy as np
+from djimaging.utils.dj_storage import load_array
+
class IplBinParamsTemplate(dj.Manual):
database = ""
@@ -26,11 +28,11 @@ class IplBinParamsTemplate(dj.Manual):
@property
def definition(self):
definition = """
- ipl_bin_id : tinyint unsigned # ipl bin id
+ ipl_bin_id : int32 # ipl bin id
---
- n_bins : tinyint unsigned # number of bins
- bin_borders :blob # pixel index where gcl/ipl border intersects on left of image (with GCL up)
- bin_names = NULL :blob # pixel index where gcl/ipl border intersects on right side of image
+ n_bins : int32 # number of bins
+ bin_borders : # pixel index where gcl/ipl border intersects on left of image (with GCL up)
+ bin_names = NULL : # pixel index where gcl/ipl border intersects on right side of image
"""
return definition
@@ -60,7 +62,7 @@ def definition(self):
-> self.roi_ipl_table
-> self.ipl_bin_params_table
---
- roi_ipl_bin : tinyint unsigned # ipl bin id -1 if too low, n_bins if too high
+ roi_ipl_bin : int32 # ipl bin id -1 if too low, n_bins if too high
"""
return definition
@@ -83,6 +85,6 @@ def key_source(self):
def make(self, key):
roi_ipl_depth = (self.roi_ipl_table & key).fetch1('ipl_depth')
- bin_borders = (self.ipl_bin_params_table & key).fetch1('bin_borders')
+ bin_borders = load_array((self.ipl_bin_params_table & key).fetch1('bin_borders'))
roi_ipl_bin = np.digitize(roi_ipl_depth, bin_borders) - 1
self.insert1(dict(**key, roi_ipl_bin=roi_ipl_bin))
diff --git a/djimaging/tables/location/location.py b/djimaging/tables/location/location.py
index 1131cff..4edff9c 100644
--- a/djimaging/tables/location/location.py
+++ b/djimaging/tables/location/location.py
@@ -53,9 +53,9 @@ def definition(self):
-> self.opticdisk_table
-> self.field_table
---
- relx :float # XCoord_um relative to the optic disk
- rely :float # YCoord_um relative to the optic disk
- relz :float # ZCoord_um relative to the optic disk
+ relx :float32 # XCoord_um relative to the optic disk
+ rely :float32 # YCoord_um relative to the optic disk
+ relz :float32 # ZCoord_um relative to the optic disk
"""
return definition
@@ -94,7 +94,7 @@ def make(self, key: dict) -> None:
absx, absy, absz = (self.field_table() & key).fetch1('absx', 'absy', 'absz')
if self.presentation_table is not None:
- pres_absx, pres_absy, pres_absz = (self.presentation_table() & key).fetch('absx', 'absy', 'absz')
+ pres_absx, pres_absy, pres_absz = (self.presentation_table() & key).to_arrays('absx', 'absy', 'absz')
if np.abs(np.median(pres_absx) - absx) > 200 or np.abs(np.median(pres_absy) - absy) > 200:
warnings.warn(f"Position is different from the presentation table for {key}. "
f"Using heuristic to determine location.")
@@ -126,7 +126,7 @@ def plot(self, view: str = 'igor_local') -> None:
Args:
view: Axis orientation; one of ``'igor_local'`` or ``'igor_setup'``.
"""
- relx, rely = self.fetch("relx", "rely")
+ relx, rely = self.to_arrays("relx", "rely")
plot_relxy_pos(relx, rely, view=view)
@@ -142,8 +142,8 @@ def definition(self):
-> self.relativefieldlocation_table
---
- ventral_dorsal_pos_um :float # position on the ventral-dorsal axis, greater 0 means dorsal
- temporal_nasal_pos_um :float # position on the temporal-nasal axis, greater 0 means nasal
+ ventral_dorsal_pos_um :float32 # position on the ventral-dorsal axis, greater 0 means dorsal
+ temporal_nasal_pos_um :float32 # position on the temporal-nasal axis, greater 0 means nasal
"""
return definition
@@ -192,7 +192,7 @@ def plot(self, key: dict = None) -> None:
key: Optional DataJoint primary key dict. If provided, the matching entry is
highlighted with a separate scatter marker.
"""
- temporal_nasal_pos_um, ventral_dorsal_pos_um = self.fetch("temporal_nasal_pos_um", "ventral_dorsal_pos_um")
+ temporal_nasal_pos_um, ventral_dorsal_pos_um = self.to_arrays("temporal_nasal_pos_um", "ventral_dorsal_pos_um")
fig, ax = plt.subplots(1, 1, figsize=(5, 5))
ax.scatter(temporal_nasal_pos_um, ventral_dorsal_pos_um, label='all')
if key is not None:
diff --git a/djimaging/tables/location/location_from_table.py b/djimaging/tables/location/location_from_table.py
index 151c3af..45eeda4 100644
--- a/djimaging/tables/location/location_from_table.py
+++ b/djimaging/tables/location/location_from_table.py
@@ -27,6 +27,7 @@ class RetinalFieldLocationFromTable(location_from_table.RetinalFieldLocationFrom
import pandas as pd
from matplotlib import pyplot as plt
+from djimaging.utils.dj_storage import open_object
from djimaging.utils.dj_utils import make_hash
@@ -39,7 +40,7 @@ def definition(self):
definition = """
table_hash : varchar(63) # hash of the classifier params config
---
- table_path : attach@{store} # Path to table
+ table_path : # Path to table
col_experimenter = 'experimenter' : varchar(191)
col_exp_num = 'exp_num' : varchar(191)
col_date = 'date' : varchar(191)
@@ -70,8 +71,8 @@ def definition(self):
-> self.field_table
-> self.params_table
---
- ventral_dorsal_pos :float # position on the ventral-dorsal axis, greater 0 means dorsal
- temporal_nasal_pos :float # position on the temporal-nasal axis, greater 0 means nasal
+ ventral_dorsal_pos :float32 # position on the ventral-dorsal axis, greater 0 means dorsal
+ temporal_nasal_pos :float32 # position on the temporal-nasal axis, greater 0 means nasal
"""
return definition
@@ -100,7 +101,8 @@ def expinfo_table(self):
def make(self, key):
params = (self.params_table() & key).fetch1()
- df = pd.read_csv(params['table_path'])
+ with open_object(params['table_path'], mode='r', store=self.params_table.store) as f:
+ df = pd.read_csv(f)
df.date = df.date.apply(lambda x: datetime.date(datetime.strptime(str(x), '%Y%m%d')))
df.head()
@@ -127,7 +129,7 @@ def make(self, key):
self.insert1(rfl_key)
def plot(self, key=None):
- temporal_nasal_pos, ventral_dorsal_pos = self.fetch("temporal_nasal_pos", "ventral_dorsal_pos")
+ temporal_nasal_pos, ventral_dorsal_pos = self.to_arrays("temporal_nasal_pos", "ventral_dorsal_pos")
fig, ax = plt.subplots(1, 1, figsize=(5, 5))
ax.scatter(temporal_nasal_pos, ventral_dorsal_pos, label='all')
if key is not None:
@@ -142,15 +144,12 @@ def plot(self, key=None):
def prepare_dj_config_location_from_table(input_folder):
stores_dict = {
- "location_table_input": {"protocol": "file", "location": input_folder, "stage": input_folder}}
+ "location_table_input": {"protocol": "file", "location": input_folder}}
# Make sure folders exits
for store, store_dict in stores_dict.items():
- for name in store_dict.keys():
- if name in ["location", "stage"]:
- assert os.path.isdir(store_dict[name]), f'This must be a folder you have access to: {store_dict[name]}'
-
- os.environ["DJ_SUPPORT_FILEPATH_MANAGEMENT"] = "TRUE"
+ assert os.path.isdir(store_dict["location"]), \
+ f'This must be a folder you have access to: {store_dict["location"]}'
dj_config_stores = dj.config['stores'] or dict()
dj_config_stores.update(stores_dict)
diff --git a/djimaging/tables/location/optic_disk.py b/djimaging/tables/location/optic_disk.py
index a32a1d5..76d0760 100644
--- a/djimaging/tables/location/optic_disk.py
+++ b/djimaging/tables/location/optic_disk.py
@@ -11,6 +11,7 @@
import numpy as np
import pandas as pd
+from djimaging.utils.dj_storage import relative_store_path
from djimaging.utils.scanm.read_h5_utils import load_h5_table
from djimaging.utils.filesystem_utils import get_file_info_df
from djimaging.utils.scanm import read_smp_utils
@@ -18,6 +19,7 @@
class OpticDiskTemplate(dj.Computed):
database = ""
+ _filepath_store = "reference"
incl_region = False
@property
@@ -31,12 +33,12 @@ def definition(self):
"""
if self.incl_region:
definition += " region :varchar(16) # region (e.g. LR or RR)\n"
- definition += """
+ definition += f"""
---
- od_fromfile :varchar(191) # File from which optic disc data was extracted
- odx :float # XCoord_um relative to the optic disk
- ody :float # YCoord_um relative to the optic disk
- odz :float # ZCoord_um relative to the optic disk
+ od_fromfile : # source acquisition file
+ odx :float32 # XCoord_um relative to the optic disk
+ ody :float32 # YCoord_um relative to the optic disk
+ odz :float32 # ZCoord_um relative to the optic disk
"""
return definition
@@ -113,7 +115,7 @@ def make(self, key):
if self.incl_region:
if self.field_table is None:
raise ValueError("Field table is not defined, but region is requested for optic disk.")
- regions = np.unique((self.field_table & key).fetch("region"))
+ regions = np.unique((self.field_table & key).to_arrays("region"))
if len(regions) == 0:
warnings.warn(f"No region found for key {key}. Populate field and try again.")
return
@@ -131,7 +133,7 @@ def make(self, key):
loc_key["odx"] = odx
loc_key["ody"] = ody
loc_key["odz"] = odz
- loc_key["od_fromfile"] = fromfile
+ loc_key["od_fromfile"] = relative_store_path(fromfile, self._filepath_store)
if self.incl_region:
if region is None:
raise ValueError(f"Region not found for key {key}. This is a new feature, so it could be a bug.")
diff --git a/djimaging/tables/location/outline.py b/djimaging/tables/location/outline.py
index 128d1c2..72995e8 100644
--- a/djimaging/tables/location/outline.py
+++ b/djimaging/tables/location/outline.py
@@ -37,6 +37,7 @@ class OutlineRelField(location.OutlineRelTemplate.OutlineRelField):
import pandas as pd
from matplotlib import pyplot as plt
+from djimaging.utils.dj_storage import load_array, relative_store_path
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.filesystem_utils import get_file_info_df
from djimaging.utils.scanm.recording import ScanMRecording
@@ -45,6 +46,7 @@ class OutlineRelField(location.OutlineRelTemplate.OutlineRelField):
class OutlineAbsTemplate(dj.Computed):
database = ""
+ _filepath_store = "reference"
_from_raw_data = True # Use SMP or h5 files?
_num_loc = 3 # In which location of filename is the number found?
@@ -53,29 +55,33 @@ def definition(self):
definition = """
-> self.experiment_table
---
- outline_abs_xy : blob # outline of the retinal field in absolute coordinates
+ outline_abs_xy : # outline of the retinal field in absolute coordinates
"""
return definition
class OutlineAbsField(dj.Part):
- definition = """
+ _filepath_store = "reference"
+
+ @property
+ def definition(self):
+ return f"""
-> master
field :varchar(32) # string identifying files corresponding to field
---
- field_data_file: varchar(191) # info extracted from which file?
- absx: float # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
- absy: float # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
- absz: float # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
- scan_type: enum("xy", "xz", "xyz") # Type of scan
- npixartifact : int unsigned # number of pixel with light artifact
- nxpix: int unsigned # number of pixels in x
- nypix: int unsigned # number of pixels in y
- nzpix: int unsigned # number of pixels in z
- nxpix_offset: int unsigned # number of offset pixels in x
- nxpix_retrace: int unsigned # number of retrace pixels in x
- pixel_size_um :float # width of a pixel in um (also height if y is second dimension)
- z_step_um = NULL :float # z-step in um
- nframes: int unsigned # number of pixels in time
+ field_data_file: # source acquisition file
+ absx: float32 # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
+ absy: float32 # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
+ absz: float32 # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
+ scan_type: enum('xy', 'xz', 'xyz') # Type of scan
+ npixartifact : int64 # number of pixel with light artifact
+ nxpix: int64 # number of pixels in x
+ nypix: int64 # number of pixels in y
+ nzpix: int64 # number of pixels in z
+ nxpix_offset: int64 # number of offset pixels in x
+ nxpix_retrace: int64 # number of retrace pixels in x
+ pixel_size_um :float32 # width of a pixel in um (also height if y is second dimension)
+ z_step_um = NULL :float32 # z-step in um
+ nframes: int64 # number of pixels in time
"""
@property
@@ -190,11 +196,10 @@ def make(self, key, verbose=False):
self.insert1(main_entry)
self.OutlineAbsField().insert(field_entries)
- @staticmethod
- def complete_key(base_key, rec) -> dict:
+ def complete_key(self, base_key, rec) -> dict:
field_entry = deepcopy(base_key)
- field_entry["field_data_file"] = rec.filepath
+ field_entry["field_data_file"] = relative_store_path(rec.filepath, self._filepath_store)
field_entry["absx"] = rec.pos_x_um
field_entry["absy"] = rec.pos_y_um
@@ -214,7 +219,7 @@ def complete_key(base_key, rec) -> dict:
def plot1(self, key=None):
key = get_primary_key(table=self, key=key)
- outline_abs_xy = (self & key).fetch1('outline_abs_xy')
+ outline_abs_xy = load_array((self & key).fetch1('outline_abs_xy'))
fig, ax = plt.subplots(1, 1, figsize=(4, 3))
ax.plot(*np.array(outline_abs_xy).T, '.-')
@@ -233,8 +238,8 @@ def definition(self):
-> self.outline_abs_table
-> self.opticdisk_table
---
- outline_rel_xy : blob # outline of the retinal field in coordinates relative to the optic disk
- outline_retina_xy : blob # outline of the retinal field in retinal coordinates
+ outline_rel_xy : # outline of the retinal field in coordinates relative to the optic disk
+ outline_retina_xy : # outline of the retinal field in retinal coordinates
"""
return definition
@@ -243,11 +248,11 @@ class OutlineRelField(dj.Part):
-> master
field :varchar(32) # string identifying files corresponding to field
---
- relx: float # relative position of the center (of the cropped field) in the x axis as recorded by ScanM
- rely: float # relative position of the center (of the cropped field) in the y axis as recorded by ScanM
- relz: float # relative position of the center (of the cropped field) in the z axis as recorded by ScanM
- ventral_dorsal_pos_um: float
- temporal_nasal_pos_um: float
+ relx: float32 # relative position of the center (of the cropped field) in the x axis as recorded by ScanM
+ rely: float32 # relative position of the center (of the cropped field) in the y axis as recorded by ScanM
+ relz: float32 # relative position of the center (of the cropped field) in the z axis as recorded by ScanM
+ ventral_dorsal_pos_um: float32
+ temporal_nasal_pos_um: float32
"""
@property
@@ -276,9 +281,9 @@ def make(self, key):
print(key)
odx, ody, odz = (self.opticdisk_table() & key).fetch1("odx", "ody", "odz")
- outline_abs_xy = (self.outline_abs_table() & key).fetch1('outline_abs_xy')
- absxs, absys, abszs, field_keys = (self.outline_abs_table().OutlineAbsField & key).fetch(
- 'absx', 'absy', 'absz', 'KEY')
+ outline_abs_xy = load_array((self.outline_abs_table() & key).fetch1('outline_abs_xy'))
+ field_keys, absxs, absys, abszs = (self.outline_abs_table().OutlineAbsField & key).to_arrays(
+ 'absx', 'absy', 'absz', include_key=True)
eye, prepwmorient = (self.expinfo_table() & key).fetch1('eye', 'prepwmorient')
@@ -322,7 +327,7 @@ def plot1(self, key=None):
outline_rel_xy = np.asarray((self & key).fetch1('outline_rel_xy'))
outline_retina_xy = np.asarray((self & key).fetch1('outline_retina_xy'))
- fields, relxs, relys = (self.OutlineRelField & key).fetch('field', 'relx', 'rely')
+ fields, relxs, relys = (self.OutlineRelField & key).to_arrays('field', 'relx', 'rely')
fig, axs = plt.subplots(1, 3, figsize=(12, 3))
diff --git a/djimaging/tables/location/roi_location.py b/djimaging/tables/location/roi_location.py
index c703bd2..4ce3596 100644
--- a/djimaging/tables/location/roi_location.py
+++ b/djimaging/tables/location/roi_location.py
@@ -33,6 +33,7 @@ class RetinalRoiLocation(location.RetinalRoiLocationTemplate):
from matplotlib import pyplot as plt
from djimaging.tables.location.location import plot_relxy_pos, plot_relyz_pos
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.scanm.setup_utils import get_retinal_position
from djimaging.utils.scanm.roi_utils import get_rel_roi_pos
@@ -50,9 +51,9 @@ def definition(self):
-> self.roi_table
---
- relx_wrt_field :float # XCoord_um relative to the field center
- rely_wrt_field :float # YCoord_um relative to the field center
- relz_wrt_field :float # ZCoord_um relative to the field center
+ relx_wrt_field :float32 # XCoord_um relative to the field center
+ rely_wrt_field :float32 # YCoord_um relative to the field center
+ relz_wrt_field :float32 # ZCoord_um relative to the field center
"""
return definition
@@ -85,10 +86,10 @@ def presentation_table(self):
def make(self, key):
roi_id = (self.roi_table & key).fetch1('roi_id')
- roi_mask = (self.roi_mask_table & key).fetch1('roi_mask')
- pixel_size_um, z_step_um, scan_type = (self.field_table & key).fetch('pixel_size_um', 'z_step_um', 'scan_type')
+ roi_mask = load_array((self.roi_mask_table & key).fetch1('roi_mask'))
+ pixel_size_um, z_step_um, scan_type = (self.field_table & key).to_arrays('pixel_size_um', 'z_step_um', 'scan_type')
- ang_deg_list = (self.presentation_table.ScanInfo() & (self.field_table & key)).fetch('angle_deg')
+ ang_deg_list = (self.presentation_table.ScanInfo() & (self.field_table & key)).to_arrays('angle_deg')
assert np.unique(ang_deg_list).size == 1, f'Found different angles for different presentations: {ang_deg_list}'
ang_deg = ang_deg_list[0]
@@ -119,7 +120,7 @@ def make(self, key):
def plot(self, restriction=None, view='igor_local'):
restriction = {} if restriction is None else restriction
- scan_type = (self.field_table & restriction).fetch('scan_type')
+ scan_type = (self.field_table & restriction).to_arrays('scan_type')
if np.unique(scan_type).size != 1:
raise ValueError(
@@ -127,7 +128,7 @@ def plot(self, restriction=None, view='igor_local'):
else:
scan_type = scan_type[0]
- relx, rely, relz = (self & restriction).fetch("relx_wrt_field", "rely_wrt_field", "relz_wrt_field")
+ relx, rely, relz = (self & restriction).to_arrays("relx_wrt_field", "rely_wrt_field", "relz_wrt_field")
if scan_type == 'xy':
plot_relxy_pos(relx, rely, view=view)
@@ -149,9 +150,9 @@ def definition(self):
-> self.relative_field_location_wrt_field_table
---
- relx :float # XCoord_um relative to the optic disk
- rely :float # YCoord_um relative to the optic disk
- relz :float # ZCoord_um relative to the optic disk
+ relx :float32 # XCoord_um relative to the optic disk
+ rely :float32 # YCoord_um relative to the optic disk
+ relz :float32 # ZCoord_um relative to the optic disk
"""
return definition
@@ -178,7 +179,7 @@ def field_table(self):
pass
def make(self, key):
- pixel_size_um, scan_type = (self.field_table & key).fetch('pixel_size_um', 'scan_type')
+ pixel_size_um, scan_type = (self.field_table & key).to_arrays('pixel_size_um', 'scan_type')
relx_wrt_field, rely_wrt_field = (self.relative_field_location_wrt_field_table & key).fetch1(
'relx_wrt_field', 'rely_wrt_field')
@@ -201,7 +202,7 @@ def make(self, key):
def plot(self, restriction=None, view='igor_local'):
restriction = {} if restriction is None else restriction
- scan_type = (self.field_table & restriction).fetch('scan_type')
+ scan_type = (self.field_table & restriction).to_arrays('scan_type')
if np.unique(scan_type).size != 1:
raise ValueError(
@@ -209,7 +210,7 @@ def plot(self, restriction=None, view='igor_local'):
else:
scan_type = scan_type[0]
- relx, rely, relz = (self & restriction).fetch("relx", "rely", "relz")
+ relx, rely, relz = (self & restriction).to_arrays("relx", "rely", "relz")
if scan_type == 'xy':
plot_relxy_pos(relx, rely, view=view)
@@ -227,8 +228,8 @@ def definition(self):
definition = """
-> self.relative_roi_location_table
---
- ventral_dorsal_pos_um :float # position on the ventral-dorsal axis, greater 0 means dorsal
- temporal_nasal_pos_um :float # position on the temporal-nasal axis, greater 0 means nasal
+ ventral_dorsal_pos_um :float32 # position on the ventral-dorsal axis, greater 0 means dorsal
+ temporal_nasal_pos_um :float32 # position on the temporal-nasal axis, greater 0 means nasal
"""
return definition
@@ -266,7 +267,7 @@ def make(self, key):
def plot(self, restriction=None, key=None):
restriction = {} if restriction is None else restriction
- temporal_nasal_pos_um, ventral_dorsal_pos_um = (self & restriction).fetch(
+ temporal_nasal_pos_um, ventral_dorsal_pos_um = (self & restriction).to_arrays(
"temporal_nasal_pos_um", "ventral_dorsal_pos_um")
fig, ax = plt.subplots(1, 1, figsize=(5, 5))
ax.scatter(temporal_nasal_pos_um, ventral_dorsal_pos_um, label='all', s=1, alpha=0.5)
diff --git a/djimaging/tables/misc/animal_age.py b/djimaging/tables/misc/animal_age.py
index 1c12c96..554e5df 100644
--- a/djimaging/tables/misc/animal_age.py
+++ b/djimaging/tables/misc/animal_age.py
@@ -15,7 +15,7 @@ def definition(self) -> str:
definition = """
-> self.experiment_table
---
- age : int # age of the animal in days
+ age : int32 # age of the animal in days
"""
return definition
@@ -83,4 +83,3 @@ def date_str_to_date(date_str: str) -> Optional[date]:
-
diff --git a/djimaging/tables/misc/brightness.py b/djimaging/tables/misc/brightness.py
index f3a3102..61b6b76 100644
--- a/djimaging/tables/misc/brightness.py
+++ b/djimaging/tables/misc/brightness.py
@@ -19,6 +19,7 @@ class RoiBrightness(misc.RoiBrightnessTemplate):
import numpy as np
from djimaging.utils import math_utils
+from djimaging.utils.dj_storage import load_array
class RoiBrightnessTemplate(dj.Computed):
@@ -34,8 +35,8 @@ def definition(self) -> str:
-> self.presentation_table
-> self.roi_table
---
- brightness_ch_data :float # For data channel defined by user
- brightness_ch_alt :float # For alternative channel defined by user
+ brightness_ch_data :float32 # For data channel defined by user
+ brightness_ch_alt :float32 # For alternative channel defined by user
"""
return definition
@@ -72,13 +73,14 @@ def make(self, key: dict) -> None:
Args:
key: DataJoint primary key dict identifying the presentation and ROI entry.
"""
- roi_mask = (self.roimask_table & key).fetch1('roi_mask')
+ roi_mask = load_array((self.roimask_table & key).fetch1('roi_mask'))
data_name, alt_name = (self.userinfo_table & key).fetch1('data_stack_name', 'alt_stack_name')
- main_ch_average = (self.presentation_table.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
+ main_ch_average = load_array(
+ (self.presentation_table.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
try:
- alt_ch_average = (self.presentation_table.StackAverages & key & f'ch_name="{alt_name}"').fetch1(
- 'ch_average')
+ alt_ch_average = load_array(
+ (self.presentation_table.StackAverages & key & dict(ch_name=alt_name)).fetch1('ch_average'))
except dj.DataJointError:
alt_ch_average = np.full_like(main_ch_average, np.nan)
main_ch_average = math_utils.normalize_zero_one(main_ch_average)
diff --git a/djimaging/tables/misc/corr_map.py b/djimaging/tables/misc/corr_map.py
index 4f67ed7..9d607e4 100644
--- a/djimaging/tables/misc/corr_map.py
+++ b/djimaging/tables/misc/corr_map.py
@@ -20,6 +20,7 @@ class CorrMap(misc.CorrMapTemplate):
from djimaging.autorois.autoshift_utils import shift_img
from djimaging.autorois.corr_roi_mask_utils import stack_corr_image
+from djimaging.utils.dj_storage import load_array, local_path
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.math_utils import normalize_zscore
from djimaging.utils.plot_utils import prep_long_title
@@ -44,14 +45,14 @@ def definition(self) -> str:
definition = """
-> self.presentation_table
---
- corr_map : longblob # Correlation mask for stack, after stimulus onset
- corr_map_max : float # Maximum correlation
- corr_map_mean : float # Mean correlation
+ corr_map : # Correlation mask for stack, after stimulus onset
+ corr_map_max : float32 # Maximum correlation
+ corr_map_mean : float32 # Mean correlation
"""
if self._include_prestim:
definition += """
- corr_map_pre_stim : longblob # Correlation mask for stack before stimulus
+ corr_map_pre_stim : # Correlation mask for stack before stimulus
"""
return definition
@@ -100,13 +101,14 @@ def make(self, key: dict) -> None:
Args:
key: DataJoint primary key dict identifying the presentation entry.
"""
- filepath = (self.presentation_table & key).fetch1('pres_data_file')
+ filepath_ref = (self.presentation_table & key).fetch1('pres_data_file')
from_raw_data = (self.raw_params_table & key).fetch1('from_raw_data')
data_name = (self.userinfo_table & key).fetch1('data_stack_name')
- triggertimes = (self.presentation_table & key).fetch1('triggertimes')
+ triggertimes = load_array((self.presentation_table & key).fetch1('triggertimes'))
fs = (self.presentation_table.ScanInfo & key).fetch1('scan_frequency')
- stack = read_utils.load_stacks(filepath, from_raw_data, ch_names=(data_name,))[0][data_name]
+ with local_path(filepath_ref, self.presentation_table._filepath_store) as filepath:
+ stack = read_utils.load_stacks(filepath, from_raw_data, ch_names=(data_name,))[0][data_name]
if len(triggertimes) > 0:
idx_start = int(triggertimes[0] * fs)
@@ -133,11 +135,12 @@ def plot1(self, key: dict = None, gamma: float = 0.7) -> None:
key = get_primary_key(self, key=key)
data_name = (self.userinfo_table & key).fetch1('data_stack_name')
- main_ch_average = (self.presentation_table.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
+ main_ch_average = load_array(
+ (self.presentation_table.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
- corr_map = (self & key).fetch1('corr_map')
+ corr_map = load_array((self & key).fetch1('corr_map'))
if self._include_prestim:
- corr_map_pre_stim = (self & key).fetch1('corr_map_pre_stim')
+ corr_map_pre_stim = load_array((self & key).fetch1('corr_map_pre_stim'))
vabsmax = np.max(np.abs(corr_map))
@@ -187,10 +190,10 @@ def definition(self) -> str:
-> self.corr_map_table.proj(cond1_A='{self._split_cond}')
-> self.corr_map_table.proj(cond1_B='{self._split_cond}')
---
- cross_corr_map : blob # Cross correlation between stacks of same stimulus but different conditions
- shift_x : int # Shift in pixels
- shift_z : int # Shift in pixels
- max_corr : float # Maximum correlation
+ cross_corr_map : # Cross correlation between stacks of same stimulus but different conditions
+ shift_x : int32 # Shift in pixels
+ shift_z : int32 # Shift in pixels
+ max_corr : float32 # Maximum correlation
"""
return definition
@@ -218,7 +221,7 @@ def make(self, key: dict, plot: bool = False) -> None:
key: DataJoint primary key dict identifying the pair of conditions.
plot: If ``True``, generate a diagnostic plot during computation.
"""
- correlation, x_shift, z_shift, max_corr = self._make_compute(key)
+ correlation, x_shift, z_shift, max_corr = self._make_compute(key, plot=plot)
self.insert1(dict(key, cross_corr_map=correlation, shift_x=x_shift, shift_z=z_shift, max_corr=max_corr))
def _make_compute(self, key: dict, plot: bool = False) -> tuple:
@@ -234,19 +237,7 @@ def _make_compute(self, key: dict, plot: bool = False) -> tuple:
key_a = {**key, 'cond1': key[f'{self._split_cond}_A']}
key_b = {**key, 'cond1': key[f'{self._split_cond}_B']}
- corr_map_a = (self.corr_map_table & key_a).fetch1('corr_map')
- corr_map_b = (self.corr_map_table & key_b).fetch1('corr_map')
-
- cut_x = self.corr_map_table().get_cut_x()
- cut_z = self.corr_map_table().get_cut_z()
-
- image1 = corr_map_a[cut_x[0]:-cut_x[1], cut_z[1]:-cut_z[0]]
- image2 = corr_map_b[cut_x[0]:-cut_x[1], cut_z[1]:-cut_z[0]]
-
- correlation, x_shift, z_shift, max_corr = cross_correlate_images(
- image1, image2, plot=plot, max_shift=self._max_shift)
-
- return correlation, x_shift, z_shift, max_corr
+ return _fetch_and_correlate_maps(self.corr_map_table, key_a, key_b, self._max_shift, plot=plot)
def plot1(self, key: dict = None) -> None:
"""Plot the cross-condition correlation map for a given key.
@@ -271,10 +262,10 @@ def definition(self) -> str:
-> self.corr_map_table.proj(stim_A='stim_name')
-> self.corr_map_table.proj(stim_B='stim_name')
---
- cross_corr_map : blob # Cross correlation between stacks of same stimulus but different conditions
- shift_x : int # Shift in pixels
- shift_z : int # Shift in pixels
- max_corr : float # Maximum correlation
+ cross_corr_map : # Cross correlation between stacks of same stimulus but different conditions
+ shift_x : int32 # Shift in pixels
+ shift_z : int32 # Shift in pixels
+ max_corr : float32 # Maximum correlation
"""
return definition
@@ -300,7 +291,7 @@ def make(self, key: dict, plot: bool = False) -> None:
key: DataJoint primary key dict identifying the pair of stimuli.
plot: If ``True``, generate a diagnostic plot during computation.
"""
- correlation, x_shift, z_shift, max_corr = self._make_compute(key)
+ correlation, x_shift, z_shift, max_corr = self._make_compute(key, plot=plot)
self.insert1(dict(key, cross_corr_map=correlation, shift_x=x_shift, shift_z=z_shift, max_corr=max_corr))
def _make_compute(self, key: dict, plot: bool = False) -> tuple:
@@ -316,19 +307,7 @@ def _make_compute(self, key: dict, plot: bool = False) -> tuple:
key_a = {**key, 'stim_name': key['stim_A']}
key_b = {**key, 'stim_name': key['stim_B']}
- corr_map_a = (self.corr_map_table & key_a).fetch1('corr_map')
- corr_map_b = (self.corr_map_table & key_b).fetch1('corr_map')
-
- cut_x = self.corr_map_table().get_cut_x()
- cut_z = self.corr_map_table().get_cut_z()
-
- image1 = corr_map_a[cut_x[0]:-cut_x[1], cut_z[1]:-cut_z[0]]
- image2 = corr_map_b[cut_x[0]:-cut_x[1], cut_z[1]:-cut_z[0]]
-
- correlation, x_shift, z_shift, max_corr = cross_correlate_images(
- image1, image2, plot=plot, max_shift=self._max_shift)
-
- return correlation, x_shift, z_shift, max_corr
+ return _fetch_and_correlate_maps(self.corr_map_table, key_a, key_b, self._max_shift, plot=plot)
def plot1(self, key: dict = None) -> None:
"""Plot the cross-stimulus correlation map for a given key.
@@ -340,6 +319,22 @@ def plot1(self, key: dict = None) -> None:
self._make_compute(key, plot=True)
+def _fetch_and_correlate_maps(
+ corr_map_table: type[CorrMapTemplate],
+ key_a: dict,
+ key_b: dict,
+ max_shift: int | None,
+ plot: bool = False,
+) -> tuple[np.ndarray, np.integer, np.integer, np.floating]:
+ """Load and crop a pair of correlation maps using the table's scan margins."""
+ corr_map_a = load_array((corr_map_table & key_a).fetch1('corr_map'))
+ corr_map_b = load_array((corr_map_table & key_b).fetch1('corr_map'))
+ cut_x = corr_map_table().get_cut_x()
+ cut_z = corr_map_table().get_cut_z()
+ crop = (slice(cut_x[0], -cut_x[1] or None), slice(cut_z[1], -cut_z[0] or None))
+ return cross_correlate_images(corr_map_a[crop], corr_map_b[crop], plot=plot, max_shift=max_shift)
+
+
def normalized_cross_correlation(image1: np.ndarray, image2: np.ndarray) -> np.ndarray:
"""Compute the normalized 2-D cross-correlation between two images.
diff --git a/djimaging/tables/misc/highresolution.py b/djimaging/tables/misc/highresolution.py
index a5cd8aa..168e4a5 100644
--- a/djimaging/tables/misc/highresolution.py
+++ b/djimaging/tables/misc/highresolution.py
@@ -22,6 +22,7 @@ class StackAverages(misc.HighResTemplate.StackAverages):
import datajoint as dj
import numpy as np
+from djimaging.utils.dj_storage import load_array, relative_store_path
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_field
from djimaging.utils.scanm.recording import ScanMRecording
@@ -31,6 +32,7 @@ class HighResTemplate(dj.Computed):
"""DataJoint computed table template for high-resolution stack metadata and averages."""
database = ""
+ _filepath_store = "reference"
_fallback_to_raw = True # If h5 not available, try to load from raw data
incl_region = True # Include region as primary key?
@@ -54,22 +56,22 @@ def definition(self) -> str:
if self.incl_cond3 and not self.field_table.incl_cond3:
definition += " cond3 :varchar(16) # condition (pharmacological or other)\n"
- definition += """
+ definition += f"""
---
- highres_file :varchar(191) # path to file (e.g. h5 file)
- absx: float # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
- absy: float # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
- absz: float # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
- scan_type: enum("xy", "xz", "xyz") # Type of scan
- npixartifact : int unsigned # number of pixel with light artifact
- nxpix: int unsigned # number of pixels in x
- nypix: int unsigned # number of pixels in y
- nzpix: int unsigned # number of pixels in z
- nxpix_offset: int unsigned # number of offset pixels in x
- nxpix_retrace: int unsigned # number of retrace pixels in x
- pixel_size_um :float # width of a pixel in um (also height if y is second dimension)
- z_step_um = NULL :float # z-step in um
- nframes: int unsigned # number of pixels in time
+ highres_file : # source acquisition file (e.g. HDF5)
+ absx: float32 # absolute position of the center (of the cropped field) in the x axis as recorded by ScanM
+ absy: float32 # absolute position of the center (of the cropped field) in the y axis as recorded by ScanM
+ absz: float32 # absolute position of the center (of the cropped field) in the z axis as recorded by ScanM
+ scan_type: enum('xy', 'xz', 'xyz') # Type of scan
+ npixartifact : int64 # number of pixel with light artifact
+ nxpix: int64 # number of pixels in x
+ nypix: int64 # number of pixels in y
+ nzpix: int64 # number of pixels in z
+ nxpix_offset: int64 # number of offset pixels in x
+ nxpix_retrace: int64 # number of retrace pixels in x
+ pixel_size_um :float32 # width of a pixel in um (also height if y is second dimension)
+ z_step_um = NULL :float32 # z-step in um
+ nframes: int64 # number of pixels in time
"""
return definition
@@ -122,7 +124,7 @@ def definition(self):
-> master
ch_name : varchar(191) # name of the channel
---
- ch_average :longblob # Stack median over time
+ ch_average : # Stack median over time
"""
return definition
@@ -221,8 +223,7 @@ def _add_entry(self, key: dict, filepath: str, setupid: str) -> None:
for avg_key in avg_entries:
(self.StackAverages & key).insert1(avg_key, allow_direct_insert=True)
- @staticmethod
- def _complete_keys(base_key: dict, rec) -> tuple:
+ def _complete_keys(self, base_key: dict, rec) -> tuple:
"""Build the high-resolution entry dict and per-channel average entry dicts.
Args:
@@ -235,7 +236,7 @@ def _complete_keys(base_key: dict, rec) -> tuple:
to insert into ``StackAverages``.
"""
hr_entry = deepcopy(base_key)
- hr_entry["highres_file"] = rec.filepath
+ hr_entry["highres_file"] = relative_store_path(rec.filepath, self._filepath_store)
hr_entry["absx"] = rec.pos_x_um
hr_entry["absy"] = rec.pos_y_um
@@ -266,9 +267,10 @@ def plot1(self, key: dict = None, figsize: tuple = (8, 4), gamma: float = 0.7) -
scan_type = (self & key).fetch1('scan_type')
data_name, alt_name = (self.userinfo_table & key).fetch1('data_stack_name', 'alt_stack_name')
- main_ch_average = (self.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
+ main_ch_average = load_array((self.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
try:
- alt_ch_average = (self.StackAverages & key & f'ch_name="{alt_name}"').fetch1('ch_average')
+ alt_ch_average = load_array(
+ (self.StackAverages & key & dict(ch_name=alt_name)).fetch1('ch_average'))
except dj.DataJointError:
alt_ch_average = np.full_like(main_ch_average, np.nan)
plot_field(main_ch_average, alt_ch_average, scan_type=scan_type,
diff --git a/djimaging/tables/misc/light_artifact.py b/djimaging/tables/misc/light_artifact.py
index b4d8ac8..5195502 100644
--- a/djimaging/tables/misc/light_artifact.py
+++ b/djimaging/tables/misc/light_artifact.py
@@ -21,6 +21,7 @@ class LightArtifact(misc.LightArtifactTemplate):
from djimaging.autorois.corr_roi_mask_utils import stack_corr_image
from djimaging.tables.core.averages import compute_upsampled_average
from djimaging.tables.core.snippets import get_aligned_snippets_times
+from djimaging.utils.dj_storage import load_array, local_path
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.math_utils import normalize
from djimaging.utils.scanm import read_utils
@@ -39,8 +40,8 @@ def definition(self) -> str:
definition = """
-> self.presentation_table
---
- light_artifact : longblob # Normalized mean light artifact trace in line precision
- triggertimes_rel : blob # Relative triggertimes
+ light_artifact : # Normalized mean light artifact trace in line precision
+ triggertimes_rel : # Relative triggertimes
"""
return definition
@@ -77,17 +78,18 @@ def make(self, key: dict) -> None:
Args:
key: DataJoint primary key dict identifying the presentation entry.
"""
- filepath = (self.presentation_table & key).fetch1('pres_data_file')
+ filepath_ref = (self.presentation_table & key).fetch1('pres_data_file')
from_raw_data = (self.raw_params_table & key).fetch1('from_raw_data')
data_name = (self.userinfo_table & key).fetch1('data_stack_name'
if self._use_main_channel else 'alt_stack_name')
- stack = read_utils.load_stacks(filepath, from_raw_data, ch_names=(data_name,))[0][data_name]
+ with local_path(filepath_ref, self.presentation_table._filepath_store) as filepath:
+ stack = read_utils.load_stacks(filepath, from_raw_data, ch_names=(data_name,))[0][data_name]
light_artifact = stack[0, :, :].T.flatten()
ntrigger_rep = (self.stimulus_table() & key).fetch1('ntrigger_rep')
- triggertimes = (self.presentation_table() & key).fetch1('triggertimes')
- line_duration = (self.presentation_table.ScanInfo & key).fetch('line_duration')
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
+ line_duration = (self.presentation_table.ScanInfo & key).to_arrays('line_duration')
times = np.arange(light_artifact.size) * line_duration
@@ -102,9 +104,9 @@ def make(self, key: dict) -> None:
def plot1(self, key: dict = None) -> None:
key = get_primary_key(self, key=key)
- light_artifact = (self & key).fetch1('light_artifact')
- line_duration = (self.presentation_table.ScanInfo & key).fetch('line_duration')
- triggertimes_rel = (self & key).fetch1('triggertimes_rel')
+ light_artifact = load_array((self & key).fetch1('light_artifact'))
+ line_duration = (self.presentation_table.ScanInfo & key).to_arrays('line_duration')
+ triggertimes_rel = load_array((self & key).fetch1('triggertimes_rel'))
time = np.arange(light_artifact.size) * line_duration
diff --git a/djimaging/tables/misc/sr_index.py b/djimaging/tables/misc/sr_index.py
index 68fb3a1..6e46cc8 100644
--- a/djimaging/tables/misc/sr_index.py
+++ b/djimaging/tables/misc/sr_index.py
@@ -25,6 +25,7 @@ class SrIndex(misc.SrIndexTemplate):
from matplotlib import pyplot as plt
from djimaging.utils import mask_format_utils
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.scanm import read_utils
from djimaging.utils.dj_utils import get_primary_key
@@ -45,7 +46,7 @@ def definition(self) -> str:
-> self.roimask_table
-> self.roi_table
---
- sr_idx = NULL : float # SR index >= 0 of ROI normalized to min of stack average and light_artifact = 1.
+ sr_idx = NULL : float32 # SR index >= 0 of ROI normalized to min of stack average and light_artifact = 1.
"""
return definition
@@ -95,11 +96,11 @@ def fetch_and_compute(self, key: dict, plot: bool = False, plot_sr_threshold: fl
A 2-tuple ``(roi_ids, sr_idxs)`` where ``roi_ids`` is an array of ROI
identifiers and ``sr_idxs`` is the corresponding array of SR index values.
"""
- roi_mask = (self.roimask_table & key).fetch1('roi_mask')
+ roi_mask = load_array((self.roimask_table & key).fetch1('roi_mask'))
try:
- triggertimes = (self.presentation_table & key).fetch1('triggertimes')
- fs = (self.presentation_table.ScanInfo & key).fetch('scan_frequency')
+ triggertimes = load_array((self.presentation_table & key).fetch1('triggertimes'))
+ fs = (self.presentation_table.ScanInfo & key).to_arrays('scan_frequency')
ntrigger_rep = (self.stimulus_table & key).fetch1('ntrigger_rep')
# Find stim onset and offset for faster computations
@@ -121,7 +122,7 @@ def fetch_and_compute(self, key: dict, plot: bool = False, plot_sr_threshold: fl
ch1_stack = ch_stacks['wDataCh1'][:, :, stim_onset_idx:stim_offset_idx]
- roi_ids = (self.roi_table & key).fetch('roi_id')
+ roi_ids = (self.roi_table & key).to_arrays('roi_id')
sr_idxs, roi_avgs, roi_avgs_means, lb, ub = compute_sr_idxs(
ch1_stack, roi_ids, roi_mask, npixartifact=npixartifact)
diff --git a/djimaging/tables/motion_correction/motion_detection.py b/djimaging/tables/motion_correction/motion_detection.py
index 8901323..138bf30 100644
--- a/djimaging/tables/motion_correction/motion_detection.py
+++ b/djimaging/tables/motion_correction/motion_detection.py
@@ -22,6 +22,7 @@ class MotionDetection(motion_correction.MotionDetectionTemplate):
from djimaging.tables.motion_correction.motion_utils import compute_shifts_jnormcorre, correct_shifts_in_stack, \
plot_stack_and_corr_stack
+from djimaging.utils.dj_storage import load_array, local_path
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import set_long_title
from djimaging.utils.scanm import read_utils
@@ -35,10 +36,10 @@ class MotionDetectionParamsTemplate(dj.Lookup):
@property
def definition(self) -> str:
definition = f"""
- mcorr_id : tinyint unsigned
+ mcorr_id : int32
---
mcorr_method : varchar(191)
- mcorr_params : longblob
+ mcorr_params :
"""
return definition
@@ -77,11 +78,11 @@ def definition(self) -> str:
-> self.presentation_table
-> self.mcorr_params_table
---
- max_shift_x : float # Maximum shift after stimulus onset in x direction
- max_shift_y : float # Maximum shift after stimulus onset in y direction
- shifts_x : mediumblob # Shift in x direction
- shifts_y : mediumblob # Shift in y direction
- idx_stim_onset : int # Index of stimulus onset
+ max_shift_x : float32 # Maximum shift after stimulus onset in x direction
+ max_shift_y : float32 # Maximum shift after stimulus onset in y direction
+ shifts_x : # Shift in x direction
+ shifts_y : # Shift in y direction
+ idx_stim_onset : int32 # Index of stimulus onset
"""
return definition
@@ -113,12 +114,12 @@ def populate(
suppress_errors: bool = False,
return_exception_objects: bool = False,
reserve_jobs: bool = False,
- order: str = "original",
- limit: int = None,
max_calls: int = None,
display_progress: bool = False,
processes: int = 1,
make_kwargs: dict = None,
+ priority: int | None = None,
+ refresh: bool | None = None,
):
"""Populate the table, enforcing single-process execution.
@@ -130,13 +131,13 @@ def populate(
suppress_errors: Passed through to ``super().populate``.
return_exception_objects: Passed through to ``super().populate``.
reserve_jobs: Passed through to ``super().populate``.
- order: Passed through to ``super().populate``.
- limit: Passed through to ``super().populate``.
max_calls: Passed through to ``super().populate``.
display_progress: Passed through to ``super().populate``.
processes: Number of parallel processes. Values greater than 1 are
reset to 1 with a warning.
make_kwargs: Passed through to ``super().populate``.
+ priority: Passed through to ``super().populate``.
+ refresh: Passed through to ``super().populate``.
"""
if processes > 1:
warnings.warn(
@@ -150,12 +151,12 @@ def populate(
suppress_errors=suppress_errors,
return_exception_objects=return_exception_objects,
reserve_jobs=reserve_jobs,
- order=order,
- limit=limit,
max_calls=max_calls,
display_progress=display_progress,
processes=processes,
make_kwargs=make_kwargs,
+ priority=priority,
+ refresh=refresh,
)
def make(self, key: dict, verbose: bool = False) -> None:
@@ -171,11 +172,13 @@ def make(self, key: dict, verbose: bool = False) -> None:
scan_frequency = (self.presentation_table.ScanInfo() & key).fetch1('scan_frequency')
pres_data_file, triggertimes, pixel_size_um, npixartifact = (self.presentation_table & key).fetch1(
'pres_data_file', 'triggertimes', 'pixel_size_um', 'npixartifact')
+ triggertimes = load_array(triggertimes)
data_stack_name = (self.userinfo_table() & key).fetch1("data_stack_name")
from_raw_data = (self.presentation_table.raw_params_table & key).fetch1('from_raw_data')
- stacks, wparams = read_utils.load_stacks(
- pres_data_file, from_raw_data=from_raw_data, ch_names=(data_stack_name,))
+ with local_path(pres_data_file, self.presentation_table._filepath_store) as filepath:
+ stacks, wparams = read_utils.load_stacks(
+ filepath, from_raw_data=from_raw_data, ch_names=(data_stack_name,))
stack = stacks[data_stack_name].copy()[npixartifact:, :]
idx_stim_onset = int(np.floor(triggertimes[0] * scan_frequency))
@@ -212,11 +215,13 @@ def plot1_stacks(self, key: dict = None, n_average: int = 10, fupsample: int = 1
fs = (self.presentation_table.ScanInfo() & key).fetch1('scan_frequency')
from_raw_data = (self.presentation_table.raw_params_table & key).fetch1('from_raw_data')
- stacks, wparams = read_utils.load_stacks(
- pres_data_file, from_raw_data=from_raw_data, ch_names=(data_stack_name,))
+ with local_path(pres_data_file, self.presentation_table._filepath_store) as filepath:
+ stacks, wparams = read_utils.load_stacks(
+ filepath, from_raw_data=from_raw_data, ch_names=(data_stack_name,))
stack = stacks[data_stack_name].copy()
shifts_x, shifts_y = (self & key).fetch1('shifts_x', 'shifts_y')
+ shifts_x, shifts_y = load_array(shifts_x), load_array(shifts_y)
stack_corrected = correct_shifts_in_stack(
stack=stack, shifts_x=shifts_x, shifts_y=shifts_y, fs=fs, fupsample=fupsample, f_cutoff=f_cutoff)
@@ -229,10 +234,11 @@ def plot1_shifts(self, key: dict = None) -> np.ndarray:
key = get_primary_key(table=self, key=key)
fs = (self.presentation_table.ScanInfo() & key).fetch1('scan_frequency')
- triggertimes = (self.presentation_table & key).fetch1('triggertimes')
+ triggertimes = load_array((self.presentation_table & key).fetch1('triggertimes'))
shifts_x, shifts_y, max_shift_x, max_shift_y, idx_stim_onset = (self & key).fetch1(
'shifts_x', 'shifts_y', 'max_shift_x', 'max_shift_y', 'idx_stim_onset')
+ shifts_x, shifts_y = load_array(shifts_x), load_array(shifts_y)
time = np.arange(shifts_x.size) / fs
diff --git a/djimaging/tables/receptivefield/__init__.py b/djimaging/tables/receptivefield/__init__.py
index 010db9a..fbd1344 100644
--- a/djimaging/tables/receptivefield/__init__.py
+++ b/djimaging/tables/receptivefield/__init__.py
@@ -4,7 +4,6 @@
from .spatial_rf import FitDoG2DRFTemplate, FitGauss2DRFTemplate
from .split_strf import SplitRFParamsTemplate, SplitRFTemplate
from .srf_contour import RfContoursParamsTemplate, RfContoursTemplate, RfContourMetricsTemplate
-from .sta import STAParamsTemplate, STATemplate
-from .fast_sta import FastStaParamsTemplate, FastStaTemplate
+from .fast_sta import FastStaParamsTemplate, FastStaTemplate, FastStaQualityTemplate
from .temporal_rf import TempRFPropertiesTemplate
from .rf_offset import RfOffsetTemplate, RfRoiOffsetTemplate
diff --git a/djimaging/tables/receptivefield/fast_sta.py b/djimaging/tables/receptivefield/fast_sta.py
index 8ef4aeb..7c326fc 100644
--- a/djimaging/tables/receptivefield/fast_sta.py
+++ b/djimaging/tables/receptivefield/fast_sta.py
@@ -16,9 +16,11 @@ class FastSta(receptivefield.FastStaTemplate):
from abc import abstractmethod
import datajoint as dj
+from matplotlib import pyplot as plt
import numpy as np
from tqdm.notebook import tqdm
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.receptive_fields.fit_rf_utils import build_design_matrix, get_rf_timing_params, compute_rf_sta
from djimaging.utils.receptive_fields.plot_rf_utils import plot_rf_frames, plot_rf_video
@@ -34,22 +36,22 @@ class FastStaParamsTemplate(dj.Lookup):
def definition(self):
definition = """
-> self.stimulus_table
- dnoise_params_id: tinyint unsigned # unique param set id
+ dnoise_params_id: int32 # unique param set id
---
- x_stimulus : longblob # Stimulus design matrix
+ x_stimulus : # Stimulus design matrix
fit_kind : varchar(191)
- fupsample_trace : tinyint unsigned # Multiplier of sampling frequency, using linear interpolation.
- fupsample_stim = 0 : tinyint unsigned # Multiplier of sampling stimulus, using repeat.
- lowpass_cutoff = 0: float # Cutoff frequency low pass filter, applied if larger 0.
- pre_blur_sigma_s = 0: float # Gaussian blur applied after low pass filter.
- post_blur_sigma_s = 0: float # Gaussian blur applied after all other steps.
- filter_dur_s_past : float # filter duration in seconds into the past
- filter_dur_s_future : float # filter duration in seconds into the future
- rf_time: longblob # time of RF, depends on dt and shift
- dt: float # Time step between frames
- shift: int # Shift of stimulus relative to trace. If negative, prediction looks into future.
- dims: blob # Dimensions of stimulus
- burn_in: int # Number of frames to burn in
+ fupsample_trace : int32 # Multiplier of sampling frequency, using linear interpolation.
+ fupsample_stim = 0 : int32 # Multiplier of sampling stimulus, using repeat.
+ lowpass_cutoff = 0: float32 # Cutoff frequency low pass filter, applied if larger 0.
+ pre_blur_sigma_s = 0: float32 # Gaussian blur applied after low pass filter.
+ post_blur_sigma_s = 0: float32 # Gaussian blur applied after all other steps.
+ filter_dur_s_past : float32 # filter duration in seconds into the past
+ filter_dur_s_future : float32 # filter duration in seconds into the future
+ rf_time: # time of RF, depends on dt and shift
+ dt: float32 # Time step between frames
+ shift: int32 # Shift of stimulus relative to trace. If negative, prediction looks into future.
+ dims: # Dimensions of stimulus
+ burn_in: int32 # Number of frames to burn in
"""
return definition
@@ -100,7 +102,7 @@ def add_default(
"""
if stim_names is None:
- stim_names = (self.stimulus_table() & "stim_family='noise'").fetch('stim_name')
+ stim_names = (self.stimulus_table() & "stim_family='noise'").to_arrays('stim_name')
key = dict(dnoise_params_id=dnoise_params_id, fit_kind=fit_kind,
fupsample_trace=fupsample_trace, fupsample_stim=fupsample_stim,
@@ -118,11 +120,12 @@ def add_default(
stim, stim_dict, ntrigger_rep, isrepeated, framerate = (self.stimulus_table() & stim_key).fetch1(
"stim_trace", "stim_dict", "ntrigger_rep", "isrepeated", "framerate")
+ stim = load_array(stim)
- if framerate > 0:
+ if framerate > 0: # This is actually the triggerrate!
dt_trigger = 1 / framerate
else:
- triggertimes = (self.presentation_table() & stim_key).fetch('triggertimes')
+ triggertimes = (self.presentation_table() & stim_key).to_arrays('triggertimes')
dt_trigger = np.median([np.median(np.diff(tti)) for tti in triggertimes])
if "ntrigger_tot" in stim_dict:
@@ -130,18 +133,25 @@ def add_default(
elif (not isrepeated) and (ntrigger_rep > 1):
n_trigger = ntrigger_rep
else: # Infer from triggertimes
- triggertimes = (self.presentation_table() & stim_key).fetch('triggertimes')
+ triggertimes = (self.presentation_table() & stim_key).to_arrays('triggertimes')
n_trigger = int(np.percentile([tti.size for tti in triggertimes], q=98))
stimtime = np.arange(n_trigger) * dt_trigger
if verbose:
print(f"Preparing design matrix for stimulus: {stim_name}: "
- f"n_trigger: {n_trigger}, dt_trigger: {dt_trigger}")
+ f"{n_trigger=}, {dt_trigger=}, {len(stimtime)=}")
x_stimulus, rf_time, dims, burn_in, shift, dt, t0 = self.prepare_stimulus(
- stim, stimtime, stim_dict.get('ntrigger_per_frame', 1), fupsample_stim,
- filter_dur_s_past, filter_dur_s_future)
+ stim=stim, triggertimes=stimtime,
+ nframes_per_trigger=stim_dict.get('nframes_per_trigger', 1),
+ fupsample_stim=fupsample_stim,
+ filter_dur_s_past=filter_dur_s_past,
+ filter_dur_s_future=filter_dur_s_future,
+ )
+
+ if verbose:
+ print(f"Stimulus design matrix has shape: {x_stimulus.shape=} with {dt=}")
new_key = stim_key.copy()
new_key['x_stimulus'] = x_stimulus
@@ -155,8 +165,11 @@ def add_default(
@staticmethod
def prepare_stimulus(
- stim: np.ndarray, triggertimes: np.ndarray, ntrigger_per_frame: int,
- fupsample_stim: int, filter_dur_s_past: float,
+ stim: np.ndarray,
+ triggertimes: np.ndarray,
+ nframes_per_trigger: int,
+ fupsample_stim: int,
+ filter_dur_s_past: float,
filter_dur_s_future: float) -> tuple:
"""Preprocess stimulus and build the design matrix.
@@ -166,8 +179,8 @@ def prepare_stimulus(
Raw stimulus array.
triggertimes : np.ndarray
Array of trigger timestamps.
- ntrigger_per_frame : int
- Number of triggers per stimulus frame.
+ nframes_per_trigger : int
+ Number of frames per trigger
fupsample_stim : int
Upsampling factor for the stimulus.
filter_dur_s_past : float
@@ -180,7 +193,10 @@ def prepare_stimulus(
tuple
Tuple of (x_stimulus, rf_time, dims, burn_in, shift, dt, t0).
"""
- stimtime, stim = preprocess_stimulus(stim, triggertimes, ntrigger_per_frame, fupsample_stim)
+ stimtime, stim = preprocess_stimulus(
+ stim, triggertimes,
+ nframes_per_trigger=nframes_per_trigger,
+ fupsample_stim=fupsample_stim)
dt, dt_rel_error = get_mean_dt(stimtime, rtol_error=np.inf, rtol_warning=0.5)
t0 = stimtime[0]
@@ -204,7 +220,7 @@ def definition(self):
-> self.params_table
-> self.traces_table
---
- rf: longblob # spatio-temporal receptive field
+ rf: # spatio-temporal receptive field
'''
return definition
@@ -248,21 +264,29 @@ def prepare_sta_params(self, *restrictions) -> dict:
NotImplementedError
If the restrictions do not select exactly one parameter set.
"""
- restrictions = [{}] if len(restrictions) == 0 else restrictions
- if len(self.params_table() & [*restrictions]) != 1:
+ restriction = dj.AndList(restrictions)
+ params_table = self.params_table() & restriction
+ if len(params_table) != 1:
raise NotImplementedError("Only one parameter set supported. Provide `restrictions` that selects one.")
(x_stimulus, dt, rf_time, burn_in, shift, kind, dims, fupsample_stim, fupsample_trace,
- fit_kind, lowpass_cutoff, pre_blur_sigma_s, post_blur_sigma_s) = (
- self.params_table & [*restrictions]).fetch1(
+ fit_kind, lowpass_cutoff, pre_blur_sigma_s, post_blur_sigma_s) = params_table.fetch1(
'x_stimulus', 'dt', 'rf_time', 'burn_in', 'shift', 'fit_kind', 'dims', 'fupsample_stim',
"fupsample_trace", "fit_kind", "lowpass_cutoff", "pre_blur_sigma_s", "post_blur_sigma_s")
- x_stimulus = np.ascontiguousarray(x_stimulus.astype(np.float32))
+ x_stimulus = np.ascontiguousarray(load_array(x_stimulus).astype(np.float32))
+ rf_time = load_array(rf_time)
+
+ # Fetch nframes_per_trigger from the stimulus table so that stimtime
+ # reconstructed in make_compute matches the design matrix built in
+ # prepare_stimulus.
+ stim_dict = (self.params_table.stimulus_table & params_table).fetch1('stim_dict')
+ nframes_per_trigger = int(stim_dict.get('nframes_per_trigger', 1))
sta_params = dict(
x_stimulus=x_stimulus, rf_time=rf_time, burn_in=burn_in, shift=shift, kind=kind, dims=dims,
dt=dt, fupsample_trace=fupsample_trace, fupsample_stim=fupsample_stim,
+ nframes_per_trigger=nframes_per_trigger,
fit_kind=fit_kind, lowpass_cutoff=lowpass_cutoff,
pre_blur_sigma_s=pre_blur_sigma_s, post_blur_sigma_s=post_blur_sigma_s,
)
@@ -275,13 +299,13 @@ def populate(
suppress_errors: bool = False,
return_exception_objects: bool = False,
reserve_jobs: bool = False,
- order: str = "original",
- limit: int | None = None,
max_calls: int | None = None,
display_progress: bool = False,
processes: int = 1,
make_kwargs: dict | None = None,
- ) -> None:
+ priority: int | None = None,
+ refresh: bool | None = None,
+ ) -> dict:
"""Populate the table, precomputing STA parameters shared across all entries.
Parameters
@@ -294,10 +318,6 @@ def populate(
If True, return exception objects instead of raising them. Default is False.
reserve_jobs : bool, optional
If True, use DataJoint job reservation. Default is False.
- order : str, optional
- Order in which to populate entries. Default is "original".
- limit : int or None, optional
- Maximum number of entries to populate. Default is None.
max_calls : int or None, optional
Maximum number of make() calls. Default is None.
display_progress : bool, optional
@@ -306,31 +326,38 @@ def populate(
Number of parallel processes. Default is 1.
make_kwargs : dict or None, optional
Additional keyword arguments passed to make(). Default is None.
+ priority : int or None, optional
+ Minimum job priority when using distributed population.
+ refresh : bool or None, optional
+ Whether to refresh the distributed job queue.
+
+ Returns
+ -------
+ dict
+ DataJoint population summary with ``success_count`` and ``error_list``.
Raises
------
NotImplementedError
If the restrictions do not select exactly one parameter set.
"""
- restrictions = [{}] if len(restrictions) == 0 else restrictions
- if len(self.params_table() & [*restrictions]) != 1:
+ if len(self.params_table() & dj.AndList(restrictions)) != 1:
raise NotImplementedError("Only one parameter set supported. Provide `restrictions` that selects one.")
- if make_kwargs is None:
- make_kwargs = dict()
+ make_kwargs = dict(make_kwargs or {})
make_kwargs['sta_params'] = self.prepare_sta_params(*restrictions)
- super().populate(
+ return super().populate(
*restrictions,
suppress_errors=suppress_errors,
return_exception_objects=return_exception_objects,
reserve_jobs=reserve_jobs,
- order=order,
- limit=limit,
max_calls=max_calls,
display_progress=display_progress,
processes=processes,
make_kwargs=make_kwargs,
+ priority=priority,
+ refresh=refresh,
)
def make_compute(self, key: dict, sta_params: dict | None = None) -> list | None:
@@ -358,6 +385,7 @@ def make_compute(self, key: dict, sta_params: dict | None = None) -> list | None
dims = sta_params['dims']
fupsample_trace = sta_params['fupsample_trace']
fupsample_stim = sta_params['fupsample_stim']
+ nframes_per_trigger = sta_params['nframes_per_trigger']
fit_kind = sta_params['fit_kind']
lowpass_cutoff = sta_params['lowpass_cutoff']
pre_blur_sigma_s = sta_params['pre_blur_sigma_s']
@@ -365,18 +393,24 @@ def make_compute(self, key: dict, sta_params: dict | None = None) -> list | None
else:
raise NotImplementedError("Only dict supported")
- triggertimes = (self.presentation_table() & key).fetch1('triggertimes')
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
+
+ # Build stimtime at the same resolution as x_stimulus: each trigger
+ # spans nframes_per_trigger stimulus frames, each further upsampled
+ # by fupsample_stim. Total upsampling factor per trigger:
+ total_upsample = nframes_per_trigger * fupsample_stim
dt_triggertimes = np.median(np.diff(triggertimes))
xidxs = np.arange(triggertimes.size + 1)
- xidxs_stim = np.repeat(xidxs, fupsample_stim) + np.tile(
- np.arange(fupsample_stim) * 1 / fupsample_stim, xidxs.size)
+ xidxs_stim = np.repeat(xidxs, total_upsample) + np.tile(
+ np.arange(total_upsample) * 1 / total_upsample, xidxs.size)
stimtime = np.interp(
- xidxs_stim, xidxs, np.append(triggertimes, triggertimes[-1] + dt_triggertimes))[:-fupsample_stim]
+ xidxs_stim, xidxs, np.append(triggertimes, triggertimes[-1] + dt_triggertimes))[:-total_upsample]
- trace_t0s, trace_dts, traces, rf_keys = (self.traces_table() & key).fetch(
- self._traces_prefix + 'trace_t0', self._traces_prefix + 'trace_dt', self._traces_prefix + 'trace', 'KEY')
+ rf_keys, trace_t0s, trace_dts, traces = (self.traces_table() & key).to_arrays(
+ self._traces_prefix + 'trace_t0', self._traces_prefix + 'trace_dt', self._traces_prefix + 'trace',
+ include_key=True)
if len(rf_keys) == 0:
print(f"No traces found for {key}")
@@ -387,10 +421,22 @@ def make_compute(self, key: dict, sta_params: dict | None = None) -> list | None
rf_entries = []
for trace, trace_t0, rf_key in tqdm(
zip(traces, trace_t0s, rf_keys), total=len(rf_keys), desc=str(key), leave=False):
+ tracetime = tracetime_base + trace_t0
rf = self._compute_rf(
- tracetime_base + trace_t0, trace, stimtime, x_stimulus,
+ tracetime, trace, stimtime, x_stimulus,
dims, fupsample_trace, fit_kind, lowpass_cutoff, pre_blur_sigma_s, post_blur_sigma_s)
- rf_entries.append({**key, **rf_key, "rf": rf})
+
+ if np.any(np.isnan(rf)):
+ print(f"Skipping {key}, RF contains NaNs. "
+ f"x_stimulus contained {np.sum(np.isnan(x_stimulus))} NaNs. "
+ f"trace contained {np.sum(np.isnan(trace))} NaNs. "
+ f"tracetime contained {np.sum(np.isnan(tracetime))} NaNs. "
+ f"[{tracetime[0]=} ... {tracetime[-1]}] and [{stimtime[0]=} ... {stimtime[-1]}]")
+ else:
+ rf_entries.append({**key, **rf_key, "rf": rf})
+
+ if len(rf_entries) == 0 and len(traces) > 0:
+ print(f"All RFs contain NaNs for {key}, skipping.")
return rf_entries
@@ -450,6 +496,10 @@ def _compute_rf(
stimtime=stimtime, trace=trace, tracetime=tracetime)
trace = finalize_trace(trace, dt, post_blur_sigma_s).astype(np.float32)
+ if trace.size < x_stimulus.shape[0] * 1 / 3:
+ print(f"WARNING: Trace doesn't cover much of the stimulus duration."
+ f"trace-size: {trace.size}, stim-size: {x_stimulus.shape[0]}")
+
assert trace.size <= x_stimulus.shape[0], f"stim-shape: {x_stimulus.shape}, trace-shape: {trace.shape}"
assert trace.dtype == x_stimulus.dtype, (trace.dtype, x_stimulus.dtype)
rf = compute_rf_sta(np.ascontiguousarray(x_stimulus[:trace.size]),
@@ -457,17 +507,26 @@ def _compute_rf(
return rf
- def plot1(self, key: dict | None = None, downsample: int = 1) -> None:
+ def plot1(self, key: dict | None = None, kind: str = 'frames', **kwargs) -> None:
"""Plot the receptive field as frames.
Parameters
----------
key : dict or None, optional
DataJoint key to restrict the table. Default is None.
- downsample : int, optional
- Downsampling factor for the frames. Default is 1.
+ kind : str, optional
+ Kind of plot to show. Currently only 'frames' is supported. Default is 'frames'.
+ **kwargs
+ Additional keyword arguments passed to the specific plot function.
"""
- self.plot1_frames(key=key, downsample=downsample)
+ if kind == 'frames':
+ self.plot1_frames(key=key, **kwargs)
+ elif kind == 'video':
+ self.plot1_video(key=key, **kwargs)
+ elif kind == 'traces':
+ self.plot1_traces(key=key, **kwargs)
+ else:
+ raise NotImplementedError(f"kind {kind} is not supported.")
def plot1_frames(self, key: dict | None = None, downsample: int = 1) -> None:
"""Plot the receptive field as individual frames.
@@ -480,8 +539,8 @@ def plot1_frames(self, key: dict | None = None, downsample: int = 1) -> None:
Downsampling factor for the frames. Default is 1.
"""
key = get_primary_key(table=self, key=key)
- rf_time = (self.params_table & key).fetch1('rf_time')
- rf = (self & key).fetch1('rf')
+ rf_time = load_array((self.params_table & key).fetch1('rf_time'))
+ rf = load_array((self & key).fetch1('rf'))
plot_rf_frames(rf, rf_time, downsample=downsample)
def plot1_video(self, key: dict | None = None, fps: int = 10):
@@ -500,6 +559,195 @@ def plot1_video(self, key: dict | None = None, fps: int = 10):
Animation object of the RF video.
"""
key = get_primary_key(table=self, key=key)
- rf_time = (self.params_table & key).fetch1('rf_time')
- rf = (self & key).fetch1('rf')
+ rf_time = load_array((self.params_table & key).fetch1('rf_time'))
+ rf = load_array((self & key).fetch1('rf'))
return plot_rf_video(rf, rf_time, fps=fps)
+
+ def plot1_traces(self, key: dict | None = None, ax=None, cmap: str = 'viridis'):
+ """Plot the spatial components of the RF as lines vs. lag.
+
+ Useful for few-pixel stimuli where the spatial structure is low-dimensional
+ (e.g. 1D bar noise or small 2D grids). Each spatial pixel is plotted as a
+ line over rf_time, color-coded by its position along the flattened spatial
+ axis.
+
+ Parameters
+ ----------
+ key : dict or None, optional
+ DataJoint key to restrict the table. Default is None.
+ ax : matplotlib.axes.Axes or None, optional
+ Axes to plot into. If None, a new figure and axes are created.
+ cmap : str, optional
+ Matplotlib colormap name used to color pixels by spatial position.
+ Default is 'viridis'.
+
+ Returns
+ -------
+ matplotlib.axes.Axes
+ The axes containing the plot.
+ """
+ key = get_primary_key(table=self, key=key)
+ rf_time = load_array((self.params_table & key).fetch1('rf_time'))
+ rf = load_array((self & key).fetch1('rf'))
+
+ # Reshape to (n_lags, n_pixels), handling both 2D and 3D+ RFs.
+ n_lags = rf.shape[0]
+ rf_flat = rf.reshape(n_lags, -1)
+ n_pixels = rf_flat.shape[1]
+
+ if ax is None:
+ _, ax = plt.subplots(1, 1, figsize=(6, 4))
+
+ colors = plt.get_cmap(cmap)(np.linspace(0, 1, n_pixels))
+ for i in range(n_pixels):
+ ax.plot(rf_time, rf_flat[:, i], color=colors[i], lw=1)
+
+ ax.axhline(0, color='k', lw=0.5, alpha=0.5)
+ ax.axvline(0, color='k', lw=0.5, alpha=0.5)
+ ax.set_xlabel('Lag (s)')
+ ax.set_ylabel('RF amplitude')
+ ax.set_title(f"RF traces ({n_pixels} pixels)")
+
+ return ax
+
+
+class FastStaQualityTemplate(dj.Computed):
+ database = ""
+ # Duration (in seconds) at the acausal end of rf_time used as the noise/baseline region.
+ # For shift <= 0 (filter into the past), this is the most-future portion of rf_time,
+ # which should contain no stimulus-driven signal.
+ _baseline_dur_s = 0.1
+
+ @property
+ def definition(self):
+ definition = """
+ # Signal-to-noise ratio for STA receptive fields (peak-to-baseline at best pixel).
+ -> self.sta_table
+ ---
+ snr: float32 # peak_amplitude / noise_std at the best spatial location
+ peak_amplitude = NULL: float32 # max(|rf|) at the best spatial location, over all time
+ peak_z = NULL : float32
+ noise_std : float32 # std of rf at the best spatial location, within the baseline window
+ n_baseline_frames: int64 # non-negative number of time bins used as baseline
+ """
+ return definition
+
+ @property
+ @abstractmethod
+ def sta_table(self):
+ pass
+
+ def make(self, key: dict) -> None:
+ rf = load_array((self.sta_table & key).fetch1('rf'))
+ rf_time = load_array((self.sta_table.params_table & key).fetch1('rf_time'))
+
+ # Flatten spatial/feature dims so rf becomes (T, S) regardless of input shape.
+ T = rf.shape[0]
+ rf_flat = rf.reshape(T, -1) # (T, S); S == 1 if rf was 1D
+ S = rf_flat.shape[1]
+
+ # Determine baseline region: acausal end of rf_time.
+ # rf_time is ordered from past to future, so the most-future bins are at the end.
+ dt = np.median(np.diff(rf_time))
+ n_baseline = max(1, int(round(self._baseline_dur_s / dt)))
+ n_baseline = min(n_baseline, T - 1) # leave at least one signal frame
+
+ # Pooled noise estimate across ALL spatial locations in the acausal tail.
+ # This decouples noise estimation from pixel selection and makes the
+ # estimate stable and (asymptotically) invariant to adding noise-only
+ # pixels via spatial padding outside the RF.
+ baseline = rf_flat[-n_baseline:, :] # (n_baseline, S)
+ noise_std = float(np.std(baseline))
+
+ # Raw peak across all space and time.
+ peak = float(np.max(np.abs(rf_flat)))
+
+ # Extreme-value correction. With S pixels and T lags of approximately
+ # independent Gaussian noise, E[max |z|] ~ sqrt(2 * log(T * S)). Dividing
+ # the raw peak-z by this expected null peak yields a statistic whose null
+ # distribution is roughly centered at 1 regardless of S (and T), so
+ # padding the stimulus with noise-only pixels no longer inflates SNR.
+ if noise_std > 0:
+ peak_z = peak / noise_std
+ null_peak = np.sqrt(2.0 * np.log(max(T * S, 2)))
+ snr = peak_z / null_peak
+ else:
+ peak_z = np.nan
+ snr = np.nan
+
+ # Argmax pixel kept only for reporting / downstream use.
+ abs_rf = np.abs(rf_flat)
+ best_pixel = int(np.argmax(abs_rf.max(axis=0)))
+
+ self.insert1({
+ **key,
+ 'snr': snr, # size-invariant: peak_z / E[max|z|] under null
+ 'peak_amplitude': peak, # raw peak (same units as rf)
+ 'peak_z': peak_z, # peak in units of pooled noise std
+ 'noise_std': noise_std, # pooled across all S pixels
+ 'n_baseline_frames': n_baseline,
+ })
+
+
+ def plot1(self, key: dict | None = None, ax=None):
+ """Plot the RF traces with the baseline window highlighted.
+
+ Shows all spatial components as lines over rf_time, shades the baseline
+ region used for noise estimation, and marks the peak amplitude. The
+ computed SNR is shown in the title.
+
+ Parameters
+ ----------
+ key : dict or None, optional
+ DataJoint key to restrict the table. Default is None.
+ ax : matplotlib.axes.Axes or None, optional
+ Axes to plot into. If None, a new figure and axes are created.
+
+ Returns
+ -------
+ matplotlib.axes.Axes
+ The axes containing the plot.
+ """
+ from matplotlib import pyplot as plt
+
+ key = get_primary_key(table=self, key=key)
+
+ snr, peak_amplitude, noise_std, n_baseline = (self & key).fetch1(
+ 'snr', 'peak_amplitude', 'noise_std', 'n_baseline_frames')
+ rf = load_array((self.sta_table & key).fetch1('rf'))
+ rf_time = load_array((self.sta_table.params_table & key).fetch1('rf_time'))
+
+ # Flatten spatial dims to (n_lags, n_pixels)
+ n_lags = rf.shape[0]
+ rf_flat = rf.reshape(n_lags, -1)
+ n_pixels = rf_flat.shape[1]
+
+ if ax is None:
+ _, ax = plt.subplots(1, 1, figsize=(6, 4))
+
+ colors = plt.get_cmap('viridis')(np.linspace(0, 1, n_pixels))
+ for i in range(n_pixels):
+ ax.plot(rf_time, rf_flat[:, i], color=colors[i], lw=1)
+
+ # Shade baseline region (last n_baseline frames, matching the make() logic)
+ ax.axvspan(rf_time[-n_baseline], rf_time[-1], color='gray', alpha=0.2,
+ label=f'baseline ({n_baseline} frames)')
+
+ # Mark peak amplitude as horizontal reference lines
+ ax.axhline(peak_amplitude, color='red', lw=0.5, ls='--', alpha=0.7)
+ ax.axhline(-peak_amplitude, color='red', lw=0.5, ls='--', alpha=0.7,
+ label=f'±peak = {peak_amplitude:.3g}')
+
+ # Noise band
+ ax.axhline(noise_std, color='k', lw=0.5, ls=':', alpha=0.5)
+ ax.axhline(-noise_std, color='k', lw=0.5, ls=':', alpha=0.5,
+ label=f'±noise std = {noise_std:.3g}')
+
+ ax.axhline(0, color='k', lw=0.5, alpha=0.5)
+ ax.axvline(0, color='k', lw=0.5, alpha=0.5)
+ ax.set_xlabel('Lag (s)')
+ ax.set_ylabel('RF amplitude')
+ ax.set_title(f"SNR = {snr:.2f}")
+ ax.legend(loc='best', fontsize=8)
+
+ return ax
diff --git a/djimaging/tables/receptivefield/glm.py b/djimaging/tables/receptivefield/glm.py
index b5f342c..6b1ddc6 100644
--- a/djimaging/tables/receptivefield/glm.py
+++ b/djimaging/tables/receptivefield/glm.py
@@ -82,28 +82,44 @@ class FitDoG2DRfGlm(receptivefield.FitDoG2DRFTemplate):
import numpy as np
from matplotlib import pyplot as plt
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.receptive_fields.plot_rf_utils import plot_rf_frames, plot_rf_video
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.receptive_fields.glm_utils import ReceptiveFieldGLM, plot_rf_summary, quality_test
+def _get_shared_model_shift(model_dict: dict) -> int:
+ """Return the common temporal shift used by all stimulus channels."""
+ shifts = model_dict['shift']
+ if not isinstance(shifts, dict):
+ return int(shifts)
+
+ channel_names = model_dict.get('channel_names') or list(shifts)
+ channel_shifts = [int(shifts[name]) for name in channel_names]
+ if not channel_shifts:
+ raise ValueError("GLM model does not contain a stimulus-channel shift")
+ if len(set(channel_shifts)) != 1:
+ raise ValueError(f"GLM channels use different shifts: {dict(zip(channel_names, channel_shifts))}")
+ return channel_shifts[0]
+
+
class RfGlmParamsTemplate(dj.Lookup):
database = ""
@property
def definition(self):
definition = """
- rf_glm_params_id: tinyint unsigned # unique param set id
+ rf_glm_params_id: int32 # unique param set id
---
- filter_dur_s_past : float # filter duration in seconds into the past
- filter_dur_s_future : float # filter duration in seconds into the future
- df_ts : blob
- df_ws : blob
- betas : blob
- kfold : tinyint unsigned
- metric = "mse": enum("mse", "corrcoef")
+ filter_dur_s_past : float32 # filter duration in seconds into the past
+ filter_dur_s_future : float32 # filter duration in seconds into the future
+ df_ts :
+ df_ws :
+ betas :
+ kfold : int32
+ metric = "mse": enum('mse', 'corrcoef')
output_nonlinearity = 'none' : varchar(63)
- other_params_dict : longblob
+ other_params_dict :
"""
return definition
@@ -156,19 +172,19 @@ def definition(self):
-> self.noise_traces_table
-> self.params_table
---
- rf: longblob # spatio-temporal receptive field
- dt: float
+ rf: # spatio-temporal receptive field
+ dt: float32
'''
if self._def_sta:
definition += '''
- rf_time: longblob
- shift: int
+ rf_time:
+ shift: int32
'''
definition += '''
- model_dict: longblob
- quality_dict: longblob
+ model_dict:
+ quality_dict:
'''
return definition
@@ -209,8 +225,9 @@ def make(self, key: dict, suppress_outputs: bool = False, clear_outputs: bool =
params = (self.params_table() & key).fetch1()
noise_dt, noise_t0, trace, stim_idxs = (self.noise_traces_table() & key).fetch1(
'noise_dt', 'noise_t0', 'trace', 'stim_idxs')
+ trace, stim_idxs = load_array(trace), load_array(stim_idxs)
assert trace.size == stim_idxs.size, "Trace and stim_idxs must have the same size."
- stim = (self.noise_traces_table.stimulus_table() & key).fetch1("stim_trace")
+ stim = load_array((self.noise_traces_table.stimulus_table() & key).fetch1("stim_trace"))
stim = stim[stim_idxs].astype(trace.dtype)
other_params_dict = params.pop('other_params_dict')
@@ -242,7 +259,7 @@ def make(self, key: dict, suppress_outputs: bool = False, clear_outputs: bool =
if self._def_sta:
rf_key['rf_time'] = model_dict.pop('rf_time')
- rf_key['shift'] = model_dict['shift']['stimulus'] # There may be other shifts
+ rf_key['shift'] = _get_shared_model_shift(model_dict)
rf_key['model_dict'] = model_dict
rf_key['quality_dict'] = quality_dict
@@ -259,6 +276,7 @@ def plot1(self, key: dict | None = None) -> None:
"""
key = get_primary_key(table=self, key=key)
rf, quality_dict, model_dict = (self & key).fetch1('rf', 'quality_dict', 'model_dict')
+ rf = load_array(rf)
plot_rf_summary(rf=rf, quality_dict=quality_dict, model_dict=model_dict,
title=f"{key['date']} {key['exp_num']} {key['field']} {key['roi_id']}")
plt.show()
@@ -275,6 +293,7 @@ def plot1_frames(self, key: dict | None = None, downsample: int = 1) -> None:
"""
key = get_primary_key(table=self, key=key)
rf, model_dict = (self & key).fetch1('rf', 'model_dict')
+ rf = load_array(rf)
plot_rf_frames(rf, model_dict['rf_time'], downsample=downsample)
def plot1_video(self, key: dict | None = None, fps: int = 10):
@@ -294,6 +313,7 @@ def plot1_video(self, key: dict | None = None, fps: int = 10):
"""
key = get_primary_key(table=self, key=key)
rf, model_dict = (self & key).fetch1('rf', 'model_dict')
+ rf = load_array(rf)
return plot_rf_video(rf, model_dict['rf_time'], fps=fps)
@@ -305,12 +325,12 @@ def definition(self):
definition = '''
-> self.glm_table
---
- corrcoef_train = NULL : float
- corrcoef_test = NULL : float
- corrcoef_dev = NULL : float
- mse_train = NULL : float
- mse_dev = NULL : float
- mse_test = NULL : float
+ corrcoef_train = NULL : float32
+ corrcoef_test = NULL : float32
+ corrcoef_dev = NULL : float32
+ mse_train = NULL : float32
+ mse_dev = NULL : float32
+ mse_test = NULL : float32
'''
return definition
@@ -353,7 +373,7 @@ def plot(self, *restrictions) -> None:
*restrictions : dict
Optional DataJoint restrictions to apply before fetching.
"""
- df_q = (self & restrictions).fetch(format='frame')
+ df_q = (self & restrictions).to_pandas()
class RfGlmQualityParamsTemplate(dj.Lookup):
@@ -362,11 +382,11 @@ class RfGlmQualityParamsTemplate(dj.Lookup):
@property
def definition(self):
definition = """
- glm_quality_params_id: smallint # unique param set id
+ glm_quality_params_id: int32 # unique param set id
---
- min_corrcoef : float
- max_mse : float
- perm_alpha : float
+ min_corrcoef : float32
+ max_mse : float32
+ perm_alpha : float32
"""
return definition
@@ -400,7 +420,7 @@ def definition(self):
# Pick best model for all parameterizations
-> self.glm_table
---
- score_test : float
+ score_test : float32
'''
return definition
@@ -429,7 +449,7 @@ def make(self, key: dict) -> None:
DataJoint primary key identifying the ROI entry to process.
"""
roi_key = key.copy()
- quality_dicts, model_dicts, param_ids = (self.glm_table & roi_key).fetch(
+ quality_dicts, model_dicts, param_ids = (self.glm_table & roi_key).to_arrays(
'quality_dict', 'model_dict', 'rf_glm_params_id')
metrics = [model_dict['metric'] for model_dict in model_dicts]
@@ -467,7 +487,7 @@ def make(self, key: dict) -> None:
def plot(self) -> None:
"""Plot histogram of selected GLM parameter IDs across all entries."""
- rf_glm_params_ids = self.fetch('rf_glm_params_id')
+ rf_glm_params_ids = self.to_arrays('rf_glm_params_id')
plt.figure()
plt.hist(rf_glm_params_ids,
bins=np.arange(np.min(rf_glm_params_ids) - 0.25, np.max(rf_glm_params_ids) + 0.5, 0.5))
@@ -484,6 +504,7 @@ def plot1(self, key: dict | None = None) -> None:
"""
key = get_primary_key(table=self, key=key)
rf, quality_dict, model_dict = (self.glm_table & key).fetch1('rf', 'quality_dict', 'model_dict')
+ rf = load_array(rf)
plot_rf_summary(rf=rf, quality_dict=quality_dict, model_dict=model_dict,
title=f"{key['date']} {key['exp_num']} {key['field']} {key['roi_id']}")
plt.show()
@@ -499,7 +520,7 @@ def definition(self):
-> self.glm_single_model_table
-> self.params_table
---
- rf_glm_qidx : float
+ rf_glm_qidx : float32
'''
return definition
@@ -584,7 +605,7 @@ def plot(self, glm_quality_params_id: int = 1) -> None:
glm_quality_params_id : int, optional
ID of the quality parameter set to plot. Default is 1.
"""
- rf_glm_qidx = (self & f"glm_quality_params_id={glm_quality_params_id}").fetch("rf_glm_qidx")
+ rf_glm_qidx = (self & f"glm_quality_params_id={glm_quality_params_id}").to_arrays("rf_glm_qidx")
fig, ax = plt.subplots(1, 1, figsize=(4, 3))
ax.hist(rf_glm_qidx)
diff --git a/djimaging/tables/receptivefield/noise_traces.py b/djimaging/tables/receptivefield/noise_traces.py
index 0841ff3..e1d0a0c 100644
--- a/djimaging/tables/receptivefield/noise_traces.py
+++ b/djimaging/tables/receptivefield/noise_traces.py
@@ -8,6 +8,7 @@
import numpy as np
from djimaging.utils.receptive_fields.preprocess_rf_utils import prepare_noise_data
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
@@ -18,14 +19,14 @@ class DNoiseTraceParamsTemplate(dj.Lookup):
def definition(self):
definition = """
-> self.stimulus_table
- dnoise_params_id: tinyint unsigned # unique param set id
+ dnoise_params_id: int32 # unique param set id
---
fit_kind : varchar(191)
- fupsample_trace : tinyint unsigned # Multiplier of sampling frequency, using linear interpolation.
- fupsample_stim = 0 : tinyint unsigned # Multiplier of sampling stimulus, using repeat.
- lowpass_cutoff = 0: float # Cutoff frequency low pass filter, applied if larger 0.
- pre_blur_sigma_s = 0: float # Gaussian blur applied after low pass filter.
- post_blur_sigma_s = 0: float # Gaussian blur applied after all other steps.
+ fupsample_trace : int32 # Multiplier of sampling frequency, using linear interpolation.
+ fupsample_stim = 0 : int32 # Multiplier of sampling stimulus, using repeat.
+ lowpass_cutoff = 0: float32 # Cutoff frequency low pass filter, applied if larger 0.
+ pre_blur_sigma_s = 0: float32 # Gaussian blur applied after low pass filter.
+ post_blur_sigma_s = 0: float32 # Gaussian blur applied after all other steps.
ref_time ='trace' : enum('trace', 'stim') # Which time to use as reference.
"""
return definition
@@ -68,7 +69,7 @@ def add_default(
"""
if stim_names is None:
- stim_names = (self.stimulus_table() & 'stim_family = "noise"').fetch('stim_name')
+ stim_names = (self.stimulus_table() & "stim_family = 'noise'").to_arrays('stim_name')
key = dict(dnoise_params_id=dnoise_params_id, fit_kind=fit_kind,
fupsample_trace=fupsample_trace, fupsample_stim=fupsample_stim,
@@ -93,11 +94,11 @@ def definition(self):
-> self.traces_table
-> self.params_table
---
- trace : longblob # Trace to fit
- stim_idxs : longblob # Stimulus frame indexes
- noise_dt : float # Time-step of time component
- noise_t0 : float # Time of first sample
- dt_rel_error : float # Maximum relative error of dts, if too large, can have unwanted effects
+ trace : # Trace to fit
+ stim_idxs : # Stimulus frame indexes
+ noise_dt : float32 # Time-step of time component
+ noise_t0 : float32 # Time of first sample
+ dt_rel_error : float32 # Maximum relative error of dts, if too large, can have unwanted effects
'''
return definition
@@ -142,9 +143,11 @@ def make(self, key: dict) -> None:
DataJoint primary key identifying the entry to compute.
"""
stim, stim_dict = (self.stimulus_table() & key).fetch1("stim_trace", "stim_dict")
- triggertimes = (self.presentation_table() & key).fetch1('triggertimes')
+ stim = load_array(stim)
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
trace_t0, trace_dt, trace = (self.traces_table() & key).fetch1(
self._traces_prefix + 'trace_t0', self._traces_prefix + 'trace_dt', self._traces_prefix + 'trace')
+ trace = load_array(trace)
fupsample_trace, fupsample_stim, fit_kind, lowpass_cutoff, pre_blur_sigma_s, post_blur_sigma_s, ref_time = (
self.params_table() & key).fetch1(
"fupsample_trace", "fupsample_stim", "fit_kind", "lowpass_cutoff",
@@ -155,7 +158,7 @@ def make(self, key: dict) -> None:
stim_idxs, trace, dt, t0, dt_rel_error = prepare_noise_data(
trace=trace, tracetime=tracetime, stim=stim_idxs, triggertimes=triggertimes,
- ntrigger_per_frame=stim_dict.get('ntrigger_per_frame', 1) if stim_dict else 1,
+ nframes_per_trigger=stim_dict.get('nframes_per_trigger', 1),
fupsample_trace=fupsample_trace, fupsample_stim=fupsample_stim, ref_time=ref_time,
fit_kind=fit_kind, lowpass_cutoff=lowpass_cutoff,
pre_blur_sigma_s=pre_blur_sigma_s, post_blur_sigma_s=post_blur_sigma_s)
@@ -184,13 +187,15 @@ def plot1(self, key: dict | None = None, xlim: tuple | None = None) -> None:
raw_trace_t0, raw_trace_dt, raw_trace = (self.traces_table() & key).fetch1(
self._traces_prefix + 'trace_t0', self._traces_prefix + 'trace_dt', self._traces_prefix + 'trace')
+ raw_trace = load_array(raw_trace)
raw_tracetime = np.arange(raw_trace.size) * raw_trace_dt + raw_trace_t0
noise_t0, noise_dt, trace, stim_idxs = (self & key).fetch1('noise_t0', 'noise_dt', 'trace', 'stim_idxs')
+ trace, stim_idxs = load_array(trace), load_array(stim_idxs)
assert trace.shape[0] == stim_idxs.shape[0], "Trace and stim have different lengths"
- stim = (self.stimulus_table() & key).fetch1("stim_trace")
+ stim = load_array((self.stimulus_table() & key).fetch1("stim_trace"))
stim = stim[stim_idxs]
tracetime = np.arange(trace.size) * noise_dt + noise_t0
diff --git a/djimaging/tables/receptivefield/rf_offset.py b/djimaging/tables/receptivefield/rf_offset.py
index c4c19e7..22a5b11 100644
--- a/djimaging/tables/receptivefield/rf_offset.py
+++ b/djimaging/tables/receptivefield/rf_offset.py
@@ -26,9 +26,9 @@ class RfRoiOffset(receptivefield.RfRoiOffsetTemplate):
import datajoint as dj
import numpy as np
from matplotlib import pyplot as plt
-from sympy.physics.vector.tests.test_printing import alpha
from djimaging.tables.receptivefield import FitGauss2DRFTemplate, FitDoG2DRFTemplate, RfContoursTemplate
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.receptive_fields.srf_contour_utils import compute_cntr_center
@@ -41,9 +41,9 @@ def definition(self):
# Computes distance to center of stimulus
-> self.rf_fit_tab
---
- rf_dx_um: float # Offset in x direction in microns (x as in the field coordinates)
- rf_dy_um: float # Offset in x direction in microns (y as in the field coordinates)
- rf_d_um: float # Euclidean distance to the center in microns
+ rf_dx_um: float32 # Offset in x direction in microns (x as in the field coordinates)
+ rf_dy_um: float32 # Offset in x direction in microns (y as in the field coordinates)
+ rf_d_um: float32 # Euclidean distance to the center in microns
"""
return definition
@@ -108,7 +108,7 @@ def fetch_and_compute(self, key: dict) -> tuple:
rf_dx_um, rf_dy_um = compute_cntr_center(srf_contours[levels[0]][0])
elif isinstance(self.rf_fit_tab(), (FitDoG2DRFTemplate, FitGauss2DRFTemplate)):
- srf = (self.rf_split_tab & key).fetch1('srf')
+ srf = load_array((self.rf_split_tab & key).fetch1('srf'))
if isinstance(self.rf_fit_tab(), FitDoG2DRFTemplate):
srf_params = (self.rf_fit_tab & key).fetch1('srf_eff_center_params')
else:
@@ -160,7 +160,7 @@ def plot(self, exp_key: dict) -> tuple:
tuple
Tuple of (fig, ax) matplotlib objects.
"""
- rf_dx_um, rf_dy_um = (self & exp_key).fetch('rf_dx_um', 'rf_dy_um')
+ rf_dx_um, rf_dy_um = (self & exp_key).to_arrays('rf_dx_um', 'rf_dy_um')
fig, ax = plt.subplots(1, 1, figsize=(5, 5))
@@ -179,8 +179,8 @@ def definition(self):
-> self.rf_offset_tab
-> self.roi_pos_wrt_field_tab
---
- relx_rf_roi_um: float
- rely_rf_roi_um: float
+ relx_rf_roi_um: float32
+ rely_rf_roi_um: float32
"""
return definition
@@ -292,7 +292,7 @@ def plot(self, restriction: dict | None = None) -> tuple:
restriction = {}
df = (self * self.roi_pos_wrt_field_tab() & restriction).proj(
- 'relx_wrt_field', 'rely_wrt_field', 'relx_rf_roi_um', 'rely_rf_roi_um').fetch(format='frame')
+ 'relx_wrt_field', 'rely_wrt_field', 'relx_rf_roi_um', 'rely_rf_roi_um').to_pandas()
df['relx_rf_wrt_field'] = df['relx_rf_roi_um'] + df['relx_wrt_field']
df['rely_rf_wrt_field'] = df['rely_rf_roi_um'] + df['rely_wrt_field']
@@ -360,12 +360,13 @@ def plot_field(self, key: dict, rf_qidx_thresh: float = 0.45, n_std: float = 1,
from matplotlib.patches import Ellipse
import seaborn as sns
- roi_mask = (self.roimask_tab & key).fetch1("roi_mask")
- pixel_size_um, npixartifact, scan_type = (self.pres_tab & key).fetch(
+ roi_mask = load_array((self.roimask_tab & key).fetch1("roi_mask"))
+ pixel_size_um, npixartifact, scan_type = (self.pres_tab & key).to_arrays(
'pixel_size_um', 'npixartifact', 'scan_type')
data_name, alt_name = (self.userinfo_tab & key).fetch1('data_stack_name', 'alt_stack_name')
- main_ch_average = (self.pres_tab.StackAverages & key & f'ch_name="{data_name}"').fetch1('ch_average')
+ main_ch_average = load_array(
+ (self.pres_tab.StackAverages & key & dict(ch_name=data_name)).fetch1('ch_average'))
stim_dict = (self.stimulus_tab & key).fetch1('stim_dict')
pix_scale_x_um, pix_scale_y_um = stim_dict['pix_scale_x_um'], stim_dict['pix_scale_y_um']
@@ -390,7 +391,7 @@ def plot_field(self, key: dict, rf_qidx_thresh: float = 0.45, n_std: float = 1,
_rois[_rois < 0] = -1
ax.contour(-_rois.T, cmap='Reds', origin='lower', extent=extent, levels=[0.99], vmin=0, vmax=1)
- roi_keys = (self.rf_fit_tab & f"rf_qidx>{rf_qidx_thresh}" & key).fetch('KEY')
+ roi_keys = (self.rf_fit_tab & f"rf_qidx>{rf_qidx_thresh}" & key).keys()
colors = sns.color_palette('Spectral', len(roi_keys))
for i, roi_key in enumerate(roi_keys):
diff --git a/djimaging/tables/receptivefield/spatial_rf.py b/djimaging/tables/receptivefield/spatial_rf.py
index 821d2a6..d19f389 100644
--- a/djimaging/tables/receptivefield/spatial_rf.py
+++ b/djimaging/tables/receptivefield/spatial_rf.py
@@ -14,6 +14,7 @@
from djimaging.utils.receptive_fields.plot_rf_utils import plot_srf_gauss_fit
from djimaging.utils.receptive_fields.spatial_rf_utils import compute_gauss_srf_area, compute_surround_index, \
compute_center_index, fit_rf_model
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_srf, prep_long_title
@@ -26,13 +27,13 @@ def definition(self):
definition = """
-> self.split_rf_table
---
- srf_fit: mediumblob
- srf_params: blob
- rf_area_um2: float # Area covered by 2 standard deviations
- rf_cdia_um: float # Circle equivalent diameter
- center_index: float # Weight and sign of center in sRF
- surround_index: float # Weight and sign of surround in sRF
- rf_qidx: float # Quality index as explained variance of the sRF estimation between 0 and 1
+ srf_fit:
+ srf_params:
+ rf_area_um2: float32 # Area covered by 2 standard deviations
+ rf_cdia_um: float32 # Circle equivalent diameter
+ center_index: float32 # Weight and sign of center in sRF
+ surround_index: float32 # Weight and sign of surround in sRF
+ rf_qidx: float32 # Quality index as explained variance of the sRF estimation between 0 and 1
"""
return definition
@@ -56,7 +57,7 @@ def stimulus_table(self):
pass
def make(self, key):
- srf = (self.split_rf_table() & key).fetch1("srf")
+ srf = load_array((self.split_rf_table() & key).fetch1("srf"))
stim_dict = (self.stimulus_table() & key).fetch1("stim_dict")
# Fit RF model
@@ -81,8 +82,9 @@ def make(self, key):
def plot1(self, key=None):
key = get_primary_key(table=self, key=key)
- srf = (self.split_rf_table() & key).fetch1("srf")
+ srf = load_array((self.split_rf_table() & key).fetch1("srf"))
srf_fit, rf_qidx = (self & key).fetch1("srf_fit", 'rf_qidx')
+ srf_fit = load_array(srf_fit)
srf_params = (self & key).fetch1("srf_params")
vabsmax = np.maximum(np.max(np.abs(srf)), np.max(np.abs(srf_fit)))
@@ -109,7 +111,7 @@ def plot1(self, key=None):
def plot(self, by=("preprocess_id",)):
columns = ["rf_qidx", "rf_cdia_um", "center_index", "surround_index"]
- df = self.proj(*columns).fetch(format='frame').reset_index()
+ df = self.proj(*columns).to_pandas().reset_index()
for params, group in df.groupby(list(by)):
print(by)
print(params)
@@ -117,7 +119,7 @@ def plot(self, by=("preprocess_id",)):
plt.show()
def plot_multiple(self):
- srf_params_list = self.fetch("srf_params")
+ srf_params_list = self.to_arrays("srf_params")
fig, ax = plt.subplots(1, 1, figsize=(5, 5))
@@ -136,17 +138,17 @@ def definition(self):
definition = """
-> self.split_rf_table
---
- srf_fit: mediumblob
- srf_center_fit: mediumblob
- srf_surround_fit: mediumblob
- srf_params: blob
- srf_eff_center: mediumblob
- srf_eff_center_params: blob
- rf_qidx: float
- rf_area_um2: float # Area covered by 2 standard deviations
- rf_cdia_um: float # Circle equivalent diameter
- center_index: float # Weight and sign of center in sRF
- surround_index: float # Weight and sign of surround in sRF
+ srf_fit:
+ srf_center_fit:
+ srf_surround_fit:
+ srf_params:
+ srf_eff_center:
+ srf_eff_center_params:
+ rf_qidx: float32
+ rf_area_um2: float32 # Area covered by 2 standard deviations
+ rf_cdia_um: float32 # Circle equivalent diameter
+ center_index: float32 # Weight and sign of center in sRF
+ surround_index: float32 # Weight and sign of surround in sRF
"""
return definition
@@ -170,7 +172,7 @@ def stimulus_table(self):
pass
def make(self, key, plot=False):
- srf = (self.split_rf_table() & key).fetch1("srf")
+ srf = load_array((self.split_rf_table() & key).fetch1("srf"))
stim_dict = (self.stimulus_table() & key).fetch1("stim_dict")
srf_fit, srf_center_fit, srf_surround_fit, srf_params, eff_polarity, qi = fit_rf_model(
@@ -220,9 +222,11 @@ def make(self, key, plot=False):
def plot1(self, key=None):
key = get_primary_key(table=self, key=key)
- srf = (self.split_rf_table() & key).fetch1("srf")
+ srf = load_array((self.split_rf_table() & key).fetch1("srf"))
srf_fit, srf_center_fit, srf_surround_fit, srf_eff_center, rf_qidx, srf_ec_params = (self & key).fetch1(
"srf_fit", 'srf_center_fit', 'srf_surround_fit', 'srf_eff_center', 'rf_qidx', 'srf_eff_center_params')
+ srf_fit, srf_center_fit, srf_surround_fit, srf_eff_center = (
+ load_array(srf_fit), load_array(srf_center_fit), load_array(srf_surround_fit), load_array(srf_eff_center))
vabsmax = np.maximum(np.max(np.abs(srf)), np.max(np.abs(srf_fit)))
@@ -259,7 +263,7 @@ def plot1(self, key=None):
def plot(self, by=("preprocess_id",)):
columns = ["rf_qidx", "rf_cdia_um", "center_index", "surround_index"]
- df = self.proj(*columns).fetch(format='frame').reset_index()
+ df = self.proj(*columns).to_pandas().reset_index()
for params, group in df.groupby(list(by)):
print(by)
print(params)
diff --git a/djimaging/tables/receptivefield/split_strf.py b/djimaging/tables/receptivefield/split_strf.py
index d58827d..d6ef61c 100644
--- a/djimaging/tables/receptivefield/split_strf.py
+++ b/djimaging/tables/receptivefield/split_strf.py
@@ -8,11 +8,24 @@
from djimaging.utils.receptive_fields.temporal_rf_utils import compute_polarity_and_peak_idxs
from djimaging.utils.receptive_fields.split_rf_utils import compute_explained_rf, resize_srf, merge_strf, split_strf
from djimaging.utils import math_utils
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_srf, plot_trf, plot_signals_heatmap
from djimaging.utils.trace_utils import sort_traces
+def fetch1_rf_time(rf_table: dj.Table | type[dj.Table], key: dict) -> np.ndarray:
+ """Load RF times from the RF row, its model dictionary, or its params table."""
+ try:
+ rf_time = (rf_table & key).fetch1('rf_time')
+ except dj.DataJointError:
+ try:
+ rf_time = (rf_table & key).fetch1('model_dict')['rf_time']
+ except dj.DataJointError:
+ rf_time = (rf_table.params_table & key).fetch1('rf_time')
+ return load_array(rf_time)
+
+
class SplitRFParamsTemplate(dj.Lookup):
database = ""
_color_idxs = None
@@ -20,20 +33,20 @@ class SplitRFParamsTemplate(dj.Lookup):
@property
def definition(self):
definition = """
- split_rf_params_id: tinyint unsigned # unique param set id
+ split_rf_params_id: int32 # unique param set id
"""
if self._color_idxs:
definition += """
- color_idx: tinyint unsigned
+ color_idx: int32
"""
definition += """
---
method : varchar(63) # Method used to split RF, currently available are SVD, STD, and MAX
- blur_std : float
- blur_npix : int unsigned
- upsample_srf_scale : int unsigned
- peak_nstd : float # How many standard deviations does a peak need to be considered peak?
- npeaks_max : int unsigned # Maximum number of peaks, ignored if zero
+ blur_std : float32
+ blur_npix : int64
+ upsample_srf_scale : int64
+ peak_nstd : float32 # How many standard deviations does a peak need to be considered peak?
+ npeaks_max : int64 # Maximum number of peaks, ignored if zero
"""
return definition
@@ -70,11 +83,11 @@ def definition(self):
-> self.rf_table
-> self.split_rf_params_table
---
- srf: longblob # spatio receptive field
- trf: longblob # temporal receptive field
- polarity : tinyint # Polarity of the RF, 1 for positive, -1 for negative
- split_qidx : float # Quality index as explained variance of the sRF tRF split between 0 and 1
- trf_peak_idxs : blob # Indexes of peaks in tRF
+ srf: # spatio receptive field
+ trf: # temporal receptive field
+ polarity : int32 # Polarity of the RF, 1 for positive, -1 for negative
+ split_qidx : float32 # Quality index as explained variance of the sRF tRF split between 0 and 1
+ trf_peak_idxs : # Indexes of peaks in tRF
'''
return definition
@@ -120,7 +133,7 @@ def _make_compute(
def make(self, key):
# Get data
- strf = (self.rf_table() & key).fetch1("rf")
+ strf = load_array((self.rf_table() & key).fetch1("rf"))
rf_time = self.fetch1_rf_time(key=key)
# Get preprocess params
@@ -151,21 +164,15 @@ def make(self, key):
self.insert(entries)
- def fetch1_rf_time(self, key):
- try:
- rf_time = (self.rf_table & key).fetch1('rf_time')
- except dj.DataJointError:
- try:
- rf_time = (self.rf_table & key).fetch1('model_dict')['rf_time']
- except dj.DataJointError:
- rf_time = (self.rf_table.params_table & key).fetch1('rf_time')
- return rf_time
+ def fetch1_rf_time(self, key: dict) -> np.ndarray:
+ return fetch1_rf_time(self.rf_table, key)
def plot1(self, key=None):
key = get_primary_key(table=self, key=key)
rf_time = self.fetch1_rf_time(key=key)
srf, trf, peak_idxs = (self & key).fetch1("srf", "trf", "trf_peak_idxs")
+ srf, trf, peak_idxs = load_array(srf), load_array(trf), load_array(peak_idxs)
fig, axs = plt.subplots(1, 2, figsize=(8, 3), sharex='col')
@@ -183,7 +190,7 @@ def plot(self, restriction=None, sort=False):
if restriction is None:
restriction = dict()
- trf = math_utils.padded_vstack((self & restriction).fetch('trf'))
+ trf = math_utils.padded_vstack((self & restriction).to_arrays('trf'))
if sort:
trf = sort_traces(trf)
diff --git a/djimaging/tables/receptivefield/srf_contour.py b/djimaging/tables/receptivefield/srf_contour.py
index 39a3cb8..bfff801 100644
--- a/djimaging/tables/receptivefield/srf_contour.py
+++ b/djimaging/tables/receptivefield/srf_contour.py
@@ -31,6 +31,7 @@ class RfContourMetrics(receptivefield.RfContourMetricsTemplate):
import numpy as np
from matplotlib import pyplot as plt
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.math_utils import normalize
from djimaging.utils.plot_utils import plot_srf, set_long_title
@@ -46,13 +47,13 @@ class RfContoursParamsTemplate(dj.Lookup):
@property
def definition(self):
definition = """
- rf_contours_params_id: int # unique param set id
+ rf_contours_params_id: int32 # unique param set id
---
- blur_std : float
- blur_npix : int unsigned
- upsample_srf_scale : int unsigned
+ blur_std : float32
+ blur_npix : int64
+ upsample_srf_scale : int64
norm_kind : enum('zscore', 'zero_one', 'amp_one', 'std_one', 'none')
- levels : blob # First level is used for area and cdia
+ levels : # First level is used for area and cdia
"""
return definition
@@ -79,14 +80,14 @@ def definition(self):
-> self.split_rf_table
-> self.rf_contours_params_table
---
- srf_contours : blob
- srf_contours_um2 : blob
- srf_contours_cdia_um : blob
- is_single_contour : tinyint unsigned # True if all levels have a single contour
- largest_contour_ratio : float # Ratio of largest contour relative to full area (for level where it's lowest)
- rf_area_um2 = NULL : float # Area of largest contour at first level
- rf_cdia_um = NULL : float # Circle equivalent diameter of largest contour at first level
- irregular_index = NULL : float # Irregularity index of largest contour at first level
+ srf_contours :
+ srf_contours_um2 :
+ srf_contours_cdia_um :
+ is_single_contour : bool # True if all levels have a single contour
+ largest_contour_ratio : float32 # Ratio of largest contour relative to full area (for level where it's lowest)
+ rf_area_um2 = NULL : float32 # Area of largest contour at first level
+ rf_cdia_um = NULL : float32 # Circle equivalent diameter of largest contour at first level
+ irregular_index = NULL : float32 # Irregularity index of largest contour at first level
"""
return definition
@@ -126,7 +127,7 @@ def _fetch_and_compute_contours(self, key, plot=False):
norm_kind, blur_std, blur_npix, upsample_srf_scale, levels = (
self.rf_contours_params_table() & key).fetch1(
"norm_kind", "blur_std", "blur_npix", "upsample_srf_scale", "levels")
- srf = (self.split_rf_table() & key).fetch1("srf")
+ srf = load_array((self.split_rf_table() & key).fetch1("srf"))
pixel_size_x_um, pixel_size_y_um = self.fetch1_pixel_size(key)
if not np.isclose(pixel_size_x_um, pixel_size_y_um):
@@ -240,9 +241,9 @@ def definition(self):
definition = """
-> self.rf_contour_table
---
- center_index: float # Weight and sign of center in sRF
- surround_index: float # Weight and sign of surround in sRF (with space to center)
- full_surround_index : float # Weight and sign of surround in sRF (with no border to center)
+ center_index: float32 # Weight and sign of center in sRF
+ surround_index: float32 # Weight and sign of surround in sRF (with space to center)
+ full_surround_index : float32 # Weight and sign of surround in sRF (with no border to center)
"""
return definition
@@ -274,7 +275,7 @@ def _fetch_and_compute(self, key, plot=False):
upsample_srf_scale, levels = (self.rf_contour_table.rf_contours_params_table() & key).fetch1(
"upsample_srf_scale", "levels")
- srf = (self.rf_contour_table.split_rf_table() & key).fetch1("srf")
+ srf = load_array((self.rf_contour_table.split_rf_table() & key).fetch1("srf"))
if upsample_srf_scale > 1:
srf = resize_srf(srf, scale=upsample_srf_scale)
diff --git a/djimaging/tables/receptivefield/sta.py b/djimaging/tables/receptivefield/sta.py
deleted file mode 100644
index 4e2721b..0000000
--- a/djimaging/tables/receptivefield/sta.py
+++ /dev/null
@@ -1,238 +0,0 @@
-"""
-Tables for spike-triggered averages (STA) receptive fields (RFs).
-Actually it's implemented for calcium event triggered averages, as we usually have calcium traces.
-
-Example usage:
-
-from djimaging.tables import receptivefield
-
-@schema
-class DNoiseTraceParams(receptivefield.DNoiseTraceParamsTemplate):
- stimulus_table = Stimulus
-
-
-@schema
-class DNoiseTrace(receptivefield.DNoiseTraceTemplate):
- presentation_table = Presentation
- stimulus_table = Stimulus
- traces_table = PreprocessTraces
- params_table = DNoiseTraceParams
-
-
-@schema
-class STAParams(receptivefield.STAParamsTemplate):
- pass
-
-
-@schema
-class STA(receptivefield.STATemplate):
- noise_traces_table = DNoiseTrace
- params_table = STAParams
-
- class DataSet(receptivefield.STATemplate.DataSet):
- pass
-
-
-@schema
-class SplitRFParams(receptivefield.SplitRFParamsTemplate):
- pass
-
-
-@schema
-class SplitRF(receptivefield.SplitRFTemplate):
- rf_table = STA
- split_rf_params_table = SplitRFParams
-
-
-@schema
-class FitGauss2DRF(receptivefield.FitGauss2DRFTemplate):
- split_rf_table = SplitRF
- stimulus_table = Stimulus
-
-
-@schema
-class FitDoG2DRF(receptivefield.FitDoG2DRFTemplate):
- split_rf_table = SplitRF
- stimulus_table = Stimulus
-
-@schema
-class TempRFProperties(receptivefield.TempRFPropertiesTemplate):
- _max_dt_future = 0.1
- split_rf_table = SplitRF
- rf_table = STA
-"""
-
-from abc import abstractmethod
-from copy import deepcopy
-
-import datajoint as dj
-import numpy as np
-
-from djimaging.utils.dj_utils import get_primary_key
-from djimaging.utils.receptive_fields.fit_rf_utils import compute_linear_rf
-from djimaging.utils.receptive_fields.plot_rf_utils import plot_rf_frames, plot_rf_video
-
-
-class STAParamsTemplate(dj.Lookup):
- database = ""
-
- @property
- def definition(self):
- definition = """
- sta_params_id: tinyint unsigned # unique param set id
- ---
- rf_method : enum("sta", "mle")
- filter_dur_s_past : float # filter duration in seconds into the past
- filter_dur_s_future : float # filter duration in seconds into the future
- frac_train : float # Fraction of data used for training in (0, 1].
- frac_dev : float # Fraction of data used for hyperparameter optimization in [0, 1).
- frac_test : float # Fraction of data used for testing [0, 1).
- store_x : enum("shape", "data") # Store x (stimulus) as data or shape (less storage)?
- store_y : enum("shape", "data") # Store y (response) as data or shape (less storage)?
- """
- return definition
-
- def add_default(
- self, sta_params_id=1, rf_method="sta", filter_dur_s_past=1., filter_dur_s_future=0.,
- frac_train=1., frac_dev=0., frac_test=0., store_x='shape', store_y='shape',
- skip_duplicates=False):
- """Add default preprocess parameter to table"""
-
- if store_x == 'data' or store_y == 'data':
- if input("Are you sure you want to store the data? This creates a lot of overhead. (y/n) ") != 'y':
- return
-
- key = dict(
- sta_params_id=sta_params_id,
- rf_method=rf_method, filter_dur_s_past=filter_dur_s_past, filter_dur_s_future=filter_dur_s_future,
- frac_train=frac_train, frac_dev=frac_dev, frac_test=frac_test,
- store_x=store_x, store_y=store_y,
- )
-
- self.insert1(key, skip_duplicates=skip_duplicates)
-
-
-class STATemplate(dj.Computed):
- database = ""
-
- @property
- def definition(self):
- definition = '''
- # Compute basic receptive fields
- -> self.noise_traces_table
- -> self.params_table
- ---
- rf: longblob # spatio-temporal receptive field
- rf_time: longblob # time of RF, depends on dt and shift
- dt: float # Time step between frames
- shift: int # Shift of stimulus relative to trace. If negative, prediction looks into future.
- '''
- return definition
-
- @property
- def key_source(self):
- try:
- return (self.params_table * self.noise_traces_table).proj()
- except (AttributeError, TypeError):
- pass
-
- @property
- @abstractmethod
- def noise_traces_table(self):
- pass
-
- @property
- @abstractmethod
- def params_table(self):
- pass
-
- class DataSet(dj.Part):
- @property
- def definition(self):
- definition = """
- -> master
- kind : enum('train', 'dev', 'test') # Data set kind
- ---
- x : longblob # Input
- y : longblob # Output
- burn_in : int unsigned # Burned output s.t. y_pred.size + burn_in == y.size
- y_pred : longblob # predicted output
- cc : float # Correlation
- mse : float # Mean Squared Error
- """
- return definition
-
- def make(self, key, verbose=False):
- # TODO: Fix overhead by precomputing the design matrix
-
- filter_dur_s_past, filter_dur_s_future, rf_method = (self.params_table() & key).fetch1(
- "filter_dur_s_past", "filter_dur_s_future", "rf_method")
- store_x, store_y = (self.params_table() & key).fetch1("store_x", "store_y")
- frac_train, frac_dev, frac_test = (self.params_table() & key).fetch1("frac_train", "frac_dev", "frac_test")
- assert np.isclose(frac_train + frac_dev + frac_test, 1.0)
-
- noise_dt, trace, stim_idxs = (self.noise_traces_table() & key).fetch1('noise_dt', 'trace', 'stim_idxs')
- stim = (self.noise_traces_table.stimulus_table() & key).fetch1("stim_trace")
- stim = stim[stim_idxs].astype(trace.dtype)
-
- rf, rf_time, rf_pred, x, y, shift = compute_linear_rf(
- dt=noise_dt, trace=trace, stim=stim, frac_train=frac_train, frac_dev=frac_dev, kind=rf_method,
- filter_dur_s_past=filter_dur_s_past, filter_dur_s_future=filter_dur_s_future,
- threshold_pred=np.all(trace >= 0), batch_size_n=6_000_000, verbose=verbose)
-
- rf_key = deepcopy(key)
- rf_key['rf'] = rf.astype(np.float32)
- rf_key['rf_time'] = rf_time.astype(np.float32)
- rf_key['dt'] = noise_dt
- rf_key['shift'] = shift
- self.insert1(rf_key)
-
- for k in x.keys():
- rf_dataset_key = deepcopy(key)
- rf_dataset_key['kind'] = k
- rf_dataset_key['burn_in'] = rf_pred['burn_in']
- rf_dataset_key['x'] = x[k].astype(np.float32) if store_x == 'data' else x[k].shape
- rf_dataset_key['y'] = y[k].astype(np.float32) if store_y == 'data' else y[k].shape
- rf_dataset_key['y_pred'] = rf_pred[f'y_pred_{k}'].astype(np.float32) \
- if store_y == 'data' else rf_pred[f'y_pred_{k}'].shape
- rf_dataset_key['cc'] = rf_pred[f'cc_{k}']
- rf_dataset_key['mse'] = rf_pred[f'mse_{k}']
- self.DataSet().insert1(rf_dataset_key)
-
- def plot1_traces(self, key=None, xlim=None):
- key = get_primary_key(table=self, key=key)
-
- from matplotlib import pyplot as plt
-
- data = (self * self.DataSet() & key).fetch()
-
- fig, axs = plt.subplots(len(data), 1, figsize=(10, 3 * len(data)), squeeze=False)
- axs = axs.flat
-
- for ax, row in zip(axs, data):
- ax.set(title=f"{row['kind']} cc={row['cc']:.2f} mse={row['mse']:.2f}", ylabel='y')
- time = np.arange(row['y'].size) * row['dt']
-
- burn_in = row['burn_in']
-
- ax.plot(time[:burn_in + 1], row['y'][:burn_in + 1], label='_', ls='--', c='C0')
- ax.plot(time[burn_in:], row['y'][burn_in:], label='data', c='C0')
- ax.plot(time[burn_in:], row['y_pred'], label='pred', alpha=0.8, c='C1')
- ax.legend(loc='upper left', bbox_to_anchor=(1, 1))
- ax.set_xlim(xlim)
-
- axs[-1].set(xlabel='Time')
- plt.tight_layout()
-
- def plot1(self, key=None, downsample=1):
- self.plot1_frames(key=key, downsample=downsample)
-
- def plot1_frames(self, key=None, downsample=1):
- key = get_primary_key(table=self, key=key)
- rf, rf_time = (self & key).fetch1('rf', 'rf_time')
- plot_rf_frames(rf, rf_time, downsample=downsample)
-
- def plot1_video(self, key=None, fps=10):
- key = get_primary_key(table=self, key=key)
- rf, rf_time = (self & key).fetch1('rf', 'rf_time')
- return plot_rf_video(rf, rf_time, fps=fps)
diff --git a/djimaging/tables/receptivefield/temporal_rf.py b/djimaging/tables/receptivefield/temporal_rf.py
index 1e03153..bd448e2 100644
--- a/djimaging/tables/receptivefield/temporal_rf.py
+++ b/djimaging/tables/receptivefield/temporal_rf.py
@@ -4,8 +4,10 @@
import numpy as np
from matplotlib import pyplot as plt
+from djimaging.tables.receptivefield.split_strf import fetch1_rf_time
from djimaging.utils.receptive_fields.temporal_rf_utils import compute_trf_transience_index, compute_half_amp_width, \
compute_main_peak_lag, compute_rel_weight_baseline
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_trf, set_long_title
@@ -20,10 +22,10 @@ def definition(self):
definition = """
-> self.split_rf_table
---
- rel_weight_baseline : float
- transience_idx : float
- half_amp_width : float
- main_peak_lag : float
+ rel_weight_baseline : float32
+ transience_idx : float32
+ half_amp_width : float32
+ main_peak_lag : float32
"""
return definition
@@ -44,19 +46,13 @@ def rf_table(self):
def split_rf_table(self):
pass
- def fetch1_rf_time(self, key):
- try:
- rf_time = (self.rf_table & key).fetch1('rf_time')
- except dj.DataJointError:
- try:
- rf_time = (self.rf_table & key).fetch1('model_dict')['rf_time']
- except dj.DataJointError:
- rf_time = (self.rf_table.params_table & key).fetch1('rf_time')
- return rf_time
+ def fetch1_rf_time(self, key: dict) -> np.ndarray:
+ return fetch1_rf_time(self.rf_table, key)
def make(self, key, plot=False):
rf_time = self.fetch1_rf_time(key=key)
trf, trf_peak_idxs = (self.split_rf_table & key).fetch1('trf', 'trf_peak_idxs')
+ trf, trf_peak_idxs = load_array(trf), load_array(trf_peak_idxs)
if np.any(rf_time[trf_peak_idxs] > self._max_dt_future):
raise ValueError(f'Peak too the future. max_dt_future={self._max_dt_future}.'
@@ -82,7 +78,7 @@ def make(self, key, plot=False):
def plot(self):
fig, axs = plt.subplots(1, 4, figsize=(12, 3))
for ax, name in zip(axs, ['rel_weight_baseline', 'transience_idx', 'half_amp_width', 'main_peak_lag']):
- ax.hist(self.fetch(name))
+ ax.hist(self.to_arrays(name))
ax.set_title(name)
plt.tight_layout()
return fig, axs
@@ -92,6 +88,7 @@ def plot1(self, key=None):
rf_time = self.fetch1_rf_time(key=key)
trf, peak_idxs = (self.split_rf_table & key).fetch1('trf', 'trf_peak_idxs')
+ trf, peak_idxs = load_array(trf), load_array(peak_idxs)
rel_weight_baseline, transience_idx, half_amp_width, main_peak_lag = (self & key).fetch1(
'rel_weight_baseline', 'transience_idx', 'half_amp_width', 'main_peak_lag')
diff --git a/djimaging/tables/response/__init__.py b/djimaging/tables/response/__init__.py
index 4898a21..5443dfc 100644
--- a/djimaging/tables/response/__init__.py
+++ b/djimaging/tables/response/__init__.py
@@ -1,13 +1,12 @@
from .chirp.chirp_qi import ChirpQITemplate
from .chirp.chirp_features_rgc import ChirpFeaturesRgcTemplate
from .chirp.chirp_features_bc import ChirpFeaturesBcTemplate
-from .chirp.chirp_surround import ChirpSurroundTemplate
-from .chirp.chirp_surround_v2 import ChirpSurroundTemplate as ChirpSurroundTemplateV2
+from .chirp.chirp_surround_v2 import ChirpSurroundTemplateV2
from .sinespot.sinespot_qi import SineSpotQITemplate
from .sinespot.sinespot_features import SineSpotFeaturesTemplate
-from .movingbar.orientation import OsDsIndexesTemplate, OsDsIndexesTemplateV1, OsDsIndexesTemplateV2
+from .movingbar.orientation import OsDsIndexesTemplate
from .csl.csl import CslMetricsTemplate
from .response_quality import RepeatQITemplate
from .presentation_quality import RepeatQIPresentationTemplate
diff --git a/djimaging/tables/response/chirp/chirp_features_bc.py b/djimaging/tables/response/chirp/chirp_features_bc.py
index 0c56357..d5fb793 100644
--- a/djimaging/tables/response/chirp/chirp_features_bc.py
+++ b/djimaging/tables/response/chirp/chirp_features_bc.py
@@ -20,6 +20,8 @@ class ChirpFeaturesBc(response.ChirpFeaturesBcTemplate):
import numpy as np
from matplotlib import pyplot as plt
+from djimaging.tables.core.averages import compute_upsampled_average
+from djimaging.tables.core.snippets import fetch_snippets_and_times
from djimaging.utils.dj_utils import get_primary_key
@@ -45,17 +47,17 @@ def definition(self):
# See Franke et al. 2017 for details, for HFi see also Baden et al. 2013
-> self.snippets_table
---
- polarity_index: float # Polarity index (POi) from Franke et al. 2017
- high_frequency_index: float # High Frequency Index (HFi) from Baden et al. 2013 and Franke et al. 2017
- transience_index: float # Response transience index (RTi) from Franke et al. 2017
- plateau_index: float # Response plateau index (RPi) from Franke et al. 2017 (corrected equation!)
- tonic_release_index: float # Tonic release index (TRi) from Franke et al. 2017
- l_freq_response : float # Low frequency response
- h_freq_response : float # High frequency response
- lh_freq_index : float # Ratio of high to low frequency response
- l_contrast_response : float # Low contrast response
- h_contrast_response : float # High contrast response
- lh_contrast_index : float # Ratio of high to low contrast response
+ polarity_index: float32 # Polarity index (POi) from Franke et al. 2017
+ high_frequency_index: float32 # High Frequency Index (HFi) from Baden et al. 2013 and Franke et al. 2017
+ transience_index: float32 # Response transience index (RTi) from Franke et al. 2017
+ plateau_index: float32 # Response plateau index (RPi) from Franke et al. 2017 (corrected equation!)
+ tonic_release_index: float32 # Tonic release index (TRi) from Franke et al. 2017
+ l_freq_response : float32 # Low frequency response
+ h_freq_response : float32 # High frequency response
+ lh_freq_index : float32 # Ratio of high to low frequency response
+ l_contrast_response : float32 # Low contrast response
+ h_contrast_response : float32 # High contrast response
+ lh_contrast_index : float32 # Ratio of high to low contrast response
'''
return definition
@@ -99,16 +101,7 @@ def compute_entry(self, key: dict, plot: bool = False) -> tuple:
plateau_index, tonic_release_index, l_freq_response, h_freq_response,
lh_freq_index, l_contrast_response, h_contrast_response, lh_contrast_index).
"""
- try:
- # Deprecated
- snippets, snippets_times, triggertimes_snippets = (self.snippets_table() & key).fetch1(
- "snippets", "snippets_times", "triggertimes_snippets")
- except dj.DataJointError:
- snippets_t0, snippets_dt, snippets, triggertimes_snippets = (self.snippets_table() & key).fetch1(
- "snippets_t0", "snippets_dt", 'snippets', 'triggertimes_snippets')
-
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
+ snippets, snippets_times, triggertimes_snippets = fetch_snippets_and_times(self.snippets_table, key)
average, average_times, _ = compute_upsampled_average(
snippets, snippets_times, triggertimes_snippets, f_resample=self._fs_resample)
@@ -724,48 +717,3 @@ def compute_contrast_response_ratio(
ax.xaxis.set_major_formatter(lambda x, pos: f"{x:.1g}\n{x / fs}s")
return low_contrast_response, high_contrast_response, lh_contrast_index
-
-
-def compute_upsampled_average(
- snippets: np.ndarray,
- snippets_times: np.ndarray,
- triggertimes_snippets: np.ndarray,
- f_resample: float = 500,
-) -> tuple:
- """Resample and average snippets at the given sampling frequency.
-
- Parameters
- ----------
- snippets : np.ndarray
- 2D array of response snippets with shape (time, trials).
- snippets_times : np.ndarray
- 2D array of timestamps for each snippet with shape (time, trials).
- triggertimes_snippets : np.ndarray
- 2D array of trigger times for each snippet with shape (n_triggers, trials).
- f_resample : float, optional
- Target sampling frequency in Hz. Default is 500.
-
- Returns
- -------
- tuple
- Tuple of (average, average_times, snippets_resampled) where average is the
- mean across trials, average_times is the time axis, and snippets_resampled
- contains all resampled trials.
- """
- dt = 1 / f_resample
- stim_dur = np.median(np.diff(triggertimes_snippets[0]))
- resampled_n = int(np.ceil(stim_dur * f_resample))
- n_reps = snippets.shape[1]
-
- average_times = np.arange(0, resampled_n) * dt
-
- snippets_resampled = np.zeros((resampled_n, n_reps))
- for rep_idx in range(n_reps):
- snippets_resampled[:, rep_idx] = np.interp(
- x=average_times,
- xp=snippets_times[:, rep_idx] - triggertimes_snippets[0, rep_idx],
- fp=snippets[:, rep_idx])
-
- average = np.mean(snippets_resampled, axis=1)
-
- return average, average_times, snippets_resampled
diff --git a/djimaging/tables/response/chirp/chirp_features_rgc.py b/djimaging/tables/response/chirp/chirp_features_rgc.py
index 40a9661..13022fb 100644
--- a/djimaging/tables/response/chirp/chirp_features_rgc.py
+++ b/djimaging/tables/response/chirp/chirp_features_rgc.py
@@ -19,6 +19,7 @@ class ChirpFeatures(response.ChirpFeaturesRgcTemplate):
from matplotlib import pyplot as plt
from scipy import signal
+from djimaging.tables.core.snippets import fetch_snippets_and_times
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.plot_utils import plot_trace_and_trigger
from djimaging.utils.trace_utils import get_mean_dt, find_closest
@@ -33,8 +34,8 @@ def definition(self):
# Computes an OnOff and a transience index based on the chirp step response
-> self.snippets_table
---
- on_off_index: float # index indicating light preference (-1 Off, 1 On)
- transience_index: float # index indicating transience of response
+ on_off_index: float32 # index indicating light preference (-1 Off, 1 On)
+ transience_index: float32 # index indicating transience of response
'''
return definition
@@ -72,16 +73,7 @@ def make(self, key: dict) -> None:
key : dict
DataJoint primary key identifying the entry to populate.
"""
- try:
- # Deprecated
- snippets, snippets_times, triggertimes_snippets = (self.snippets_table() & key).fetch1(
- "snippets", "snippets_times", "triggertimes_snippets")
- except dj.DataJointError:
- snippets_t0, snippets_dt, snippets, triggertimes_snippets = (self.snippets_table() & key).fetch1(
- "snippets_t0", "snippets_dt", 'snippets', 'triggertimes_snippets')
-
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
+ snippets, snippets_times, triggertimes_snippets = fetch_snippets_and_times(self.snippets_table, key)
on_off_index = compute_on_off_index(snippets, snippets_times, triggertimes_snippets[0])
transience_index = compute_transience_index(snippets, snippets_times, triggertimes_snippets[0])
@@ -98,16 +90,7 @@ def plot1(self, key: dict | None = None) -> None:
"""
key = get_primary_key(table=self, key=key)
- try:
- # Deprecated
- snippets, snippets_times, triggertimes_snippets = (self.snippets_table() & key).fetch1(
- "snippets", "snippets_times", "triggertimes_snippets")
- except dj.DataJointError:
- snippets_t0, snippets_dt, snippets, triggertimes_snippets = (self.snippets_table() & key).fetch1(
- "snippets_t0", "snippets_dt", 'snippets', 'triggertimes_snippets')
-
- snippets_times = (np.tile(np.arange(snippets.shape[0]) * snippets_dt, (len(snippets_t0), 1)).T
- + snippets_t0)
+ snippets, snippets_times, triggertimes_snippets = fetch_snippets_and_times(self.snippets_table, key)
on_off_index, transience_index = (self & key).fetch1('on_off_index', 'transience_index')
@@ -130,7 +113,7 @@ def plot(self, restriction: dict | None = None) -> None:
if restriction is None:
restriction = dict()
- on_off_index, transience_index = (self & restriction).fetch('on_off_index', 'transience_index')
+ on_off_index, transience_index = (self & restriction).to_arrays('on_off_index', 'transience_index')
fig, axs = plt.subplots(1, 2, figsize=(8, 3))
ax = axs[0]
diff --git a/djimaging/tables/response/chirp/chirp_surround.py b/djimaging/tables/response/chirp/chirp_surround.py
deleted file mode 100644
index f9c3165..0000000
--- a/djimaging/tables/response/chirp/chirp_surround.py
+++ /dev/null
@@ -1,322 +0,0 @@
-"""
-Example usage:
-@schema
-class ChirpSurround(response.ChirpSurroundTemplate):
- snippets_table = Snippets
-"""
-
-from abc import abstractmethod
-
-import datajoint as dj
-import numpy as np
-
-from matplotlib import pyplot as plt
-
-from djimaging.utils.dj_utils import get_primary_key, get_secondary_keys
-from djimaging.tables.core.averages import compute_upsampled_average
-
-
-class ChirpSurroundTemplate(dj.Computed):
- database = ""
-
- _normalize_amp = False
- _fs_resample = 500
- _t_on_step = 2
- _t_off_step = 5
- _t_max = 6
-
- @property
- def definition(self):
- definition = f"""
- # Correlation between two response for two stimuli like local and global chirp
- -> self.snippets_table.proj(lChirp='stim_name')
- -> self.snippets_table.proj(gChirp='stim_name')
- ---
- l_polarity_index : float
- g_polarity_index : float
- surround_strength = NULL : float
- """
- return definition
-
- @property
- @abstractmethod
- def snippets_table(self):
- pass
-
- def _l_table(self):
- """Return the local chirp (lChirp) snippet table projection.
-
- Returns
- -------
- datajoint.expression.QueryExpression
- Projected snippets table restricted to lChirp stimulus.
- """
- secondary_keys = get_secondary_keys(self.snippets_table)
- return (self.snippets_table & dict(stim_name="lChirp")).proj(*secondary_keys, lChirp='stim_name')
-
- def _g_table(self):
- """Return the global chirp (gChirp) snippet table projection.
-
- Returns
- -------
- datajoint.expression.QueryExpression
- Projected snippets table restricted to gChirp stimulus.
- """
- secondary_keys = get_secondary_keys(self.snippets_table)
- return (self.snippets_table & dict(stim_name="gChirp")).proj(*secondary_keys, gChirp='stim_name')
-
- @property
- def key_source(self):
- try:
- return self._l_table().proj() * self._g_table().proj()
- except (AttributeError, TypeError):
- pass
-
- def compute_entry(self, key: dict, plot: bool = False) -> tuple:
- """Compute surround strength and polarity indices for lChirp and gChirp.
-
- Parameters
- ----------
- key : dict
- DataJoint primary key identifying the entry to compute.
- plot : bool, optional
- If True, create diagnostic plots. Default is False.
-
- Returns
- -------
- tuple
- Tuple of (surround_strength, l_polarity_index, g_polarity_index).
- """
- l_snippets_t0, l_snippets_dt, l_snippets, l_triggertimes_snippets = (self._l_table() & key).fetch1(
- "snippets_t0", "snippets_dt", 'snippets', 'triggertimes_snippets')
- g_snippets_t0, g_snippets_dt, g_snippets, g_triggertimes_snippets = (self._g_table() & key).fetch1(
- "snippets_t0", "snippets_dt", 'snippets', 'triggertimes_snippets')
-
- l_snippets_times = (np.tile(np.arange(l_snippets.shape[0]) * l_snippets_dt, (len(l_snippets_t0), 1)).T
- + l_snippets_t0)
- g_snippets_times = (np.tile(np.arange(g_snippets.shape[0]) * g_snippets_dt, (len(g_snippets_t0), 1)).T
- + g_snippets_t0)
-
- l_average, l_average_times, _ = compute_upsampled_average(
- l_snippets, l_snippets_times, l_triggertimes_snippets, f_resample=self._fs_resample)
- g_average, g_average_times, _ = compute_upsampled_average(
- g_snippets, g_snippets_times, g_triggertimes_snippets, f_resample=self._fs_resample)
-
- if plot:
- fig, axs = plt.subplots(1, 4, figsize=(12, 2))
- ax = axs[0]
- ax.set_title(f"Average trace; fs={self._fs_resample} Hz")
- ax.plot(l_average)
- ax.plot(g_average)
- ax.xaxis.set_major_formatter(lambda x, pos: f"{x:.1g}\n{x / self._fs_resample}s")
- else:
- axs = [None] * 4
-
- # Normalize as in Franke et al. 2017
- if self._normalize_amp:
- l_average /= np.max(np.abs(l_average))
- l_average = l_average - np.median(l_average[:int(self._fs_resample * self._t_on_step)])
-
- if self._normalize_amp:
- g_average /= np.max(np.abs(g_average))
- g_average = g_average - np.median(g_average[:int(self._fs_resample * self._t_on_step)])
-
- l_avg_on, l_avg_off, l_polarity_index = compute_step_response_index(
- average=l_average, fs=self._fs_resample, t_on_step=self._t_on_step, t_off_step=self._t_off_step,
- plot=axs[1], color='C0')
-
- g_avg_on, g_avg_off, g_polarity_index = compute_step_response_index(
- average=g_average, fs=self._fs_resample, t_on_step=self._t_on_step, t_off_step=self._t_off_step,
- plot=axs[2], color='C1')
-
- surround_strength = compute_surround_strength(
- lchirp_on=l_avg_on, gchirp_on=g_avg_on, lchirp_off=l_avg_off, gchirp_off=g_avg_off,
- lchirp_polarity_index=l_polarity_index, gchirp_polarity_index=g_polarity_index, plot=axs[3])
-
- if plot:
- plt.tight_layout()
- plt.show()
-
- return surround_strength, l_polarity_index, g_polarity_index
-
- def make(self, key: dict, plot: bool = False) -> None:
- """Compute and insert surround strength and polarity indices into the table.
-
- Fetches lChirp and gChirp snippet data, computes the local and global polarity
- indices and the surround strength, and inserts the result into the table.
-
- Parameters
- ----------
- key : dict
- DataJoint primary key identifying the entry to populate.
- plot : bool, optional
- If True, create diagnostic plots during computation. Default is False.
- """
- surround_strength, l_polarity_index, g_polarity_index = self.compute_entry(key, plot=plot)
-
- self.insert1(dict(
- key,
- surround_strength=surround_strength,
- l_polarity_index=l_polarity_index,
- g_polarity_index=g_polarity_index,
- ))
-
- def plot1(self, key: dict | None = None) -> None:
- """Plot diagnostic figures and verify against stored values.
-
- Parameters
- ----------
- key : dict or None, optional
- DataJoint primary key. If None, uses the first available key.
-
- Raises
- ------
- ValueError
- If computed values do not match the stored values in the table.
- """
- key = get_primary_key(table=self, key=key)
- surround_strength, l_polarity_index, g_polarity_index = \
- (self & key).fetch1('surround_strength', 'l_polarity_index', 'g_polarity_index')
-
- plot_values = np.array(self.compute_entry(key, plot=True))
- db_values = np.array([surround_strength, l_polarity_index, g_polarity_index])
-
- if not np.all(plot_values == db_values):
- raise ValueError("Computed values do not match stored values")
-
-
-def compute_step_response_index(
- average: np.ndarray,
- fs: float,
- alpha: float = 2,
- t_on_step: float = 2,
- t_off_step: float = 5,
- color: str | None = None,
- plot: plt.Axes | bool | None = None,
-) -> tuple:
- """Calculate step response and the polarity index (POi) from Franke et al. 2017.
-
- Note: In the paper it's not clear how they treated negative values, we clip them to zero.
-
- # TODO: Merge with compute_polarity_index
-
- Parameters
- ----------
- average : np.ndarray
- A 1D array of the average response trace.
- fs : float
- Sampling rate of the data in Hz.
- alpha : float, optional
- Duration of the response window in seconds. Default is 2.
- t_on_step : float, optional
- Time of the on-step of the stimulus in seconds. Default is 2.
- t_off_step : float, optional
- Time of the off-step of the stimulus in seconds. Default is 5.
- color : str or None, optional
- Matplotlib color for the trace plot. Default is None.
- plot : matplotlib.axes.Axes or bool or None, optional
- If a matplotlib Axes instance, plot into it. If True, create a new figure.
- If None or False, no plot is produced. Default is None.
-
- Returns
- -------
- tuple
- Tuple of (avg_on, avg_off, polarity_index).
- """
-
- idx_on_start = int(np.floor(t_on_step * fs))
- idx_on_end = int(np.floor((t_on_step + alpha) * fs))
-
- idx_off_start = int(np.floor(t_off_step * fs))
- idx_off_end = int(np.floor((t_off_step + alpha) * fs))
-
- avg_on = np.clip(np.mean(average[idx_on_start:idx_on_end]), 0, None)
- avg_off = np.clip(np.mean(average[idx_off_start:idx_off_end]), 0, None)
-
- scale = avg_on + avg_off
- if scale < 1e-9:
- polarity_index = 0.
- else:
- polarity_index = (avg_on - avg_off) / scale
-
- if plot:
- if isinstance(plot, plt.Axes):
- ax = plot
- else:
- fig, ax = plt.subplots(1, 1, figsize=(4, 2))
- ax.set_title(f"POi: {polarity_index:.2f}")
- ax.plot(average, color=color)
- ax.plot([idx_on_start, idx_on_end], [avg_on, avg_on], color='r')
- ax.plot([idx_off_start, idx_off_end], [avg_off, avg_off], color='r')
- ax.set_xlim(0, fs * (t_off_step + 2 * alpha))
- ax.xaxis.set_major_formatter(lambda x, pos: f"{x:.1g}\n{x / fs}s")
-
- return avg_on, avg_off, polarity_index
-
-
-def compute_surround_strength(
- lchirp_on: float,
- gchirp_on: float,
- lchirp_off: float,
- gchirp_off: float,
- lchirp_polarity_index: float,
- gchirp_polarity_index: float,
- plot: plt.Axes | bool | None = None,
-) -> float:
- """Calculate the surround strength (SSt) from Hsiang et al. 2024.
-
- Note: In the paper they only calculated the surround strength for On bipolar cells.
-
- Parameters
- ----------
- lchirp_on : float
- Light increment response of the local chirp stimulus.
- gchirp_on : float
- Light increment response of the global chirp stimulus.
- lchirp_off : float
- Light decrement response of the local chirp stimulus.
- gchirp_off : float
- Light decrement response of the global chirp stimulus.
- lchirp_polarity_index : float
- Polarity index of the local chirp stimulus.
- gchirp_polarity_index : float
- Polarity index of the global chirp stimulus.
- plot : matplotlib.axes.Axes or bool or None, optional
- If a matplotlib Axes instance, plot into it. If True, create a new figure.
- If None or False, no plot is produced. Default is None.
-
- Returns
- -------
- float
- Surround strength (SSt), or np.nan if polarity indices have opposite signs
- or the local response is too small.
- """
-
- if (lchirp_polarity_index > 0) & (gchirp_polarity_index > 0):
- if lchirp_on < 1e-5:
- surround_strength = np.nan
- else:
- surround_strength = (lchirp_on - gchirp_on) / lchirp_on
- elif (lchirp_polarity_index < 0) & (gchirp_polarity_index < 0):
- if lchirp_off < 1e-5:
- surround_strength = np.nan
- else:
- surround_strength = (lchirp_off - gchirp_off) / np.maximum(lchirp_off, 1e-9)
- else:
- surround_strength = np.nan
-
- if plot:
- if isinstance(plot, plt.Axes):
- ax = plot
- else:
- fig, ax = plt.subplots(1, 1, figsize=(4, 2))
- ax.set_title(f"SSt: {surround_strength:.2f}")
- ax.plot(0, lchirp_on, marker='^', color='C0', label='lChirp On')
- ax.plot(1, lchirp_off, marker='v', color='C0', label='lChirp Off')
- ax.plot(2, gchirp_on, marker='^', color='C1', label='gChirp On')
- ax.plot(3, gchirp_off, marker='v', color='C1', label='gChirp Off')
- ax.set_xticks([0, 1, 2, 3])
- ax.set_xlim(-0.5, 10)
- ax.legend()
-
- return surround_strength
diff --git a/djimaging/tables/response/chirp/chirp_surround_v2.py b/djimaging/tables/response/chirp/chirp_surround_v2.py
index c2fa43d..a943639 100644
--- a/djimaging/tables/response/chirp/chirp_surround_v2.py
+++ b/djimaging/tables/response/chirp/chirp_surround_v2.py
@@ -1,7 +1,7 @@
"""
Example usage:
@schema
-class ChirpSurround(response.ChirpSurroundTemplate):
+class ChirpSurroundV2(response.ChirpSurroundTemplateV2):
_l_name = 'lChirp'
_g_name = 'gChirp'
@@ -20,11 +20,12 @@ class ChirpSurround(response.ChirpSurroundTemplate):
from matplotlib import pyplot as plt
-from djimaging.utils.dj_utils import get_primary_key, get_secondary_keys
+from djimaging.tables.core.snippets import fetch_snippets_and_times
+from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.trace_utils import get_mean_dt
-class ChirpSurroundTemplate(dj.Computed):
+class ChirpSurroundTemplateV2(dj.Computed):
database = ""
_l_name = 'lChirp'
@@ -42,9 +43,9 @@ def definition(self):
-> self.snippets_table.proj({self._l_name}='stim_name')
-> self.snippets_table.proj({self._g_name}='stim_name')
---
- chirp_surround_index = NULL : float
- l_response_mus : blob # (repetitions,) local response means
- g_response_mus : blob # (repetitions,) global response means
+ chirp_surround_index = NULL : float32
+ l_response_mus : # (repetitions,) local response means
+ g_response_mus : # (repetitions,) global response means
"""
return definition
@@ -92,15 +93,8 @@ def compute_entry(self, key: dict, plot: bool = False) -> tuple:
polarity_index = (self.chirp_features_table & l_key).fetch1('polarity_index')
- l_snippets_t0, l_snippets_dt, l_snippets, l_triggertimes_snippets = (self.snippets_table() & l_key).fetch1(
- "snippets_t0", "snippets_dt", 'snippets', 'triggertimes_snippets')
- g_snippets_t0, g_snippets_dt, g_snippets, g_triggertimes_snippets = (self.snippets_table() & g_key).fetch1(
- "snippets_t0", "snippets_dt", 'snippets', 'triggertimes_snippets')
-
- l_snippets_times = (np.tile(np.arange(l_snippets.shape[0]) * l_snippets_dt, (len(l_snippets_t0), 1)).T
- + l_snippets_t0)
- g_snippets_times = (np.tile(np.arange(g_snippets.shape[0]) * g_snippets_dt, (len(g_snippets_t0), 1)).T
- + g_snippets_t0)
+ l_snippets, l_snippets_times, l_triggertimes_snippets = fetch_snippets_and_times(self.snippets_table, l_key)
+ g_snippets, g_snippets_times, g_triggertimes_snippets = fetch_snippets_and_times(self.snippets_table, g_key)
dt = get_mean_dt(l_snippets_times.T)[0]
dt_g = get_mean_dt(g_snippets_times.T)[0]
diff --git a/djimaging/tables/response/csl/csl.py b/djimaging/tables/response/csl/csl.py
index 2e3a3a5..dfb1fc8 100644
--- a/djimaging/tables/response/csl/csl.py
+++ b/djimaging/tables/response/csl/csl.py
@@ -43,6 +43,7 @@ class CslMetrics(response.CslMetricsTemplate):
from djimaging.tables.core.preprocesstraces import process_trace
from djimaging.tables.response.csl.naka_rushton_utils import fit_naka_rushton
from djimaging.tables.response.csl.sigmoid_utils import fit_sigmoid
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.math_utils import normalize_zero_one
from djimaging.utils.snippet_utils import split_trace_by_reps
@@ -73,25 +74,25 @@ def definition(self):
# Normalized contrast step light response
-> self.traces_table
---
- average: longblob # Average of the normalized snippets (time, )
- snippets: longblob # Baseline corrected snippets (times x repetitions)
- fs: float # Sampling frequency in which average and snippets are stored
- fs_metrics: float # Sampling frequency used to compute metrics
- qidx_full: float # Quality index for full trace (as in Baden et al 2016)
- qidx_contrast: float # Quality index for contrast steps only (i.e. excluding half-steps)
- on_off_index: float # Index indicating light preference (-1 Off, 1 On)
- contrast_sensitivity: float # Relating step responses to contrast responses
- tonic_release_index: float # Tonic release index as in Franke et al 2017, but for last contrast step
- plateau_index: float # Plateau index (a - b) / (a + b), similar to Franke et al 2017
- contrast_{self._metric_kind}s: blob # Metric per contrast (e.g., auc or fft_f1), incl. baseline at i=0 if _w_zero_fit=1
+ average: # Average of the normalized snippets (time, )
+ snippets: # Baseline corrected snippets (times x repetitions)
+ fs: float32 # Sampling frequency in which average and snippets are stored
+ fs_metrics: float32 # Sampling frequency used to compute metrics
+ qidx_full: float32 # Quality index for full trace (as in Baden et al 2016)
+ qidx_contrast: float32 # Quality index for contrast steps only (i.e. excluding half-steps)
+ on_off_index: float32 # Index indicating light preference (-1 Off, 1 On)
+ contrast_sensitivity: float32 # Relating step responses to contrast responses
+ tonic_release_index: float32 # Tonic release index as in Franke et al 2017, but for last contrast step
+ plateau_index: float32 # Plateau index (a - b) / (a + b), similar to Franke et al 2017
+ contrast_{self._metric_kind}s: # Metric per contrast (e.g., auc or fft_f1), incl. baseline at i=0 if _w_zero_fit=1
'''
# Add an optional phases field when using FFT metric
if getattr(self, '_metric_kind', 'auc') == 'fft_f1':
- definition += ' contrast_fft_f1_phases: blob # Phase of F1 (degrees) per contrast; baseline at i=0 if present\n'
- definition += f''' fit_half_amp_y = NULL : float # y at half amplitude of fit
- fit_half_amp_x = NULL : float # x at half amplitude of fit
- fit_half_amp_slope = NULL : float # Slope at half amplitude of fit
- droppedlastrep_flag: tinyint unsigned # Was the last repetition incomplete and therefore dropped?
+ definition += ' contrast_fft_f1_phases: # Phase of F1 (degrees) per contrast; baseline at i=0 if present\n'
+ definition += f''' fit_half_amp_y = NULL : float32 # y at half amplitude of fit
+ fit_half_amp_x = NULL : float32 # x at half amplitude of fit
+ fit_half_amp_slope = NULL : float32 # Slope at half amplitude of fit
+ droppedlastrep_flag: bool # Was the last repetition incomplete and therefore dropped?
'''
return definition
@@ -135,6 +136,7 @@ def _make_fetch_and_compute(self, key: dict, plot: bool = False) -> dict:
average, snippets, quality indices, response indices, and fit parameters.
"""
trace_dt, trace_t0, trace = (self.traces_table & key).fetch1('trace_dt', 'trace_t0', 'trace')
+ trace = load_array(trace)
if len(trace) == 0:
raise ValueError(f'Cannot compute CSL metrics for empty trace with key={key}')
@@ -143,7 +145,7 @@ def _make_fetch_and_compute(self, key: dict, plot: bool = False) -> dict:
scan_type, nypix, nzpix = (self.presentation_table & key).fetch1('scan_type', 'nypix', 'nzpix')
n_lines = int(nzpix if scan_type == 'xz' else nypix)
- triggertimes = (self.presentation_table & key).fetch1('triggertimes')
+ triggertimes = load_array((self.presentation_table & key).fetch1('triggertimes'))
ntrigger_rep = (self.stimulus_table & key).fetch1('ntrigger_rep')
fs_resample = 1 / line_duration
@@ -200,7 +202,7 @@ def make(self, key: dict, plot: bool = False, verbose: bool = False, DEBUG: bool
if verbose:
print(f'Populating {key}')
- trace = (self.traces_table & key).fetch1('trace')
+ trace = load_array((self.traces_table & key).fetch1('trace'))
if len(trace) == 0:
if verbose:
print(f'Skipping CSL metrics for empty trace with key={key}')
diff --git a/djimaging/tables/response/movingbar/orientation.py b/djimaging/tables/response/movingbar/orientation.py
index d1a035b..fe62117 100644
--- a/djimaging/tables/response/movingbar/orientation.py
+++ b/djimaging/tables/response/movingbar/orientation.py
@@ -1,5 +1,5 @@
"""
-Moving Bar feature extraction similar but not the same as in Baden et al. 2016
+Moving-bar feature extraction following the Baden et al. 2016 MATLAB implementation
Example usage:
@@ -13,16 +13,16 @@ class OsDsIndexes(response.OsDsIndexesTemplate):
stimulus_table = Stimulus
snippets_table = Snippets
"""
-import warnings
from abc import abstractmethod
import datajoint as dj
import numpy as np
from matplotlib import pyplot as plt
-from djimaging.tables.response.movingbar.orientation_utils import preprocess_mb_snippets, T_START, T_CHANGE, T_END
-from djimaging.tables.response.movingbar.orientation_utils_v1 import compute_os_ds_idxs as compute_os_ds_idxs_v1
-from djimaging.tables.response.movingbar.orientation_utils_v2 import compute_os_ds_idxs as compute_os_ds_idxs_v2
+from djimaging.tables.response.movingbar.orientation_utils import (
+ compute_os_ds_idxs, preprocess_mb_snippets, T_START, T_CHANGE, T_END,
+)
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
@@ -30,7 +30,6 @@ class OsDsIndexesTemplate(dj.Computed):
database = ""
_reduced_storage = True # Don't save all intermediate results
_n_shuffles = 100 # Number of shuffles for null distribution
- _version = 1 # or 2
@property
def definition(self):
@@ -39,26 +38,26 @@ def definition(self):
#as well as a quality index of DS responses as described in Baden et al. (2016)
-> self.snippets_table
---
- ds_index: float # direction selectivity index as resulting vector length (absolute of projection on complex exponential)
- ds_pvalue: float # p-value indicating the percentile of the vector length in null distribution
- pref_dir: float # preferred direction
- os_index: float # orientation selectivity index in analogy to ds_index
- os_pvalue: float # analogous to ds_pvalue for orientation tuning
- pref_or: float # preferred orientation
- on_off: float # on off index based on time kernel
- d_qi: float # quality index for moving bar response
- dir_component: blob
- time_component: blob
- time_component_dt: float
- surrogate_v: blob # computed by projecting on time
- surrogate_dsi: float # DSI of surrogate v
+ ds_index: float32 # direction selectivity index as resulting vector length (absolute of projection on complex exponential)
+ ds_pvalue: float32 # p-value indicating the percentile of the vector length in null distribution
+ pref_dir: float32 # preferred direction
+ os_index: float32 # orientation selectivity index in analogy to ds_index
+ os_pvalue: float32 # analogous to ds_pvalue for orientation tuning
+ pref_or: float32 # preferred orientation
+ on_off: float32 # on off index based on time kernel
+ d_qi: float32 # quality index for moving bar response
+ dir_component:
+ time_component:
+ time_component_dt: float32
+ surrogate_v: # computed by projecting on time
+ surrogate_dsi: float32 # DSI of surrogate v
"""
if not self._reduced_storage:
definition += """
- ds_null: blob # null distribution of DSIs
- os_null: blob # null distribution of OSIs
- avg_sorted_resp: longblob
+ ds_null: # null distribution of DSIs
+ os_null: # null distribution of OSIs
+ avg_sorted_resp:
"""
return definition
@@ -82,16 +81,10 @@ def key_source(self):
pass
def make(self, key):
- dir_order = (self.stimulus_table() & key).fetch1('trial_info')
+ trial_info = (self.stimulus_table() & key).fetch1('trial_info')
+ dir_order = np.asarray([trial['name'] for trial in trial_info])
snippets_dt, snippets = (self.snippets_table() & key).fetch1('snippets_dt', 'snippets')
-
- # Pick version
- if self._version == 1:
- compute_os_ds_idxs = compute_os_ds_idxs_v1
- elif self._version == 2:
- compute_os_ds_idxs = compute_os_ds_idxs_v2
- else:
- raise ValueError(f"Version {self._version} not supported.")
+ snippets = load_array(snippets)
dsi, p_dsi, null_dist_dsi, pref_dir, osi, p_osi, null_dist_osi, pref_or, \
on_off, d_qi, time_component, dir_component, surrogate_v, dsi_s, avg_sorted_responses = \
@@ -117,13 +110,15 @@ def make(self, key):
def plot1(self, key=None):
key = get_primary_key(table=self, key=key)
- dir_order = (self.stimulus_table() & key).fetch1('trial_info')
+ trial_info = (self.stimulus_table() & key).fetch1('trial_info')
+ dir_order = np.asarray([trial['name'] for trial in trial_info])
sorted_directions_rad = np.deg2rad(np.sort(dir_order))
(time_component_dt, dir_component, ds_index, ds_pvalue, os_index, os_pvalue, pref_dir, pref_or, on_off) = (
- self & key).fetch1(
+ self & key).fetch1(
'time_component_dt', 'dir_component', 'ds_index', 'ds_pvalue', 'os_index', 'os_pvalue',
'pref_dir', 'pref_or', 'on_off')
+ dir_component = load_array(dir_component)
fig, axs = plt.subplots(3, 3, figsize=(6, 6), facecolor='w', sharex=True, sharey=True)
@@ -148,9 +143,9 @@ def plot1(self, key=None):
dir_idxs = [3, 2, 1, 4, 0, 5, 6, 7]
if not self._reduced_storage:
- avg_sorted_resp = (self & key).fetch1('avg_sorted_resp')
+ avg_sorted_resp = load_array((self & key).fetch1('avg_sorted_resp'))
else:
- snippets = (self.snippets_table() & key).fetch1('snippets')
+ snippets = load_array((self.snippets_table() & key).fetch1('snippets'))
sorted_directions, sorted_responses, avg_sorted_resp = preprocess_mb_snippets(snippets, dir_order)
for idx, (ax_idx, dir_idx) in enumerate(zip(ax_idxs, dir_idxs)):
@@ -176,47 +171,9 @@ def plot(self, restriction=None):
fig, axs = plt.subplots(1, len(var_names), figsize=(len(var_names) * 2, 2), squeeze=False)
axs = axs.flatten()
for ax, var_name in zip(axs, var_names):
- dat = (self & restriction).fetch(var_name)
+ dat = (self & restriction).to_arrays(var_name)
ax.hist(dat)
ax.set(title=var_name)
plt.tight_layout()
plt.show()
return fig, axs
-
-
-class OsDsIndexesTemplateV1(OsDsIndexesTemplate):
- _version = 1
-
- def __init__(self, *args, **kwargs):
- warnings.warn("OsDsIndexesTemplateV1 is deprecated. Use OsDsIndexesTemplate with _version=1 instead.",
- DeprecationWarning, stacklevel=2)
- super().__init__(*args, **kwargs)
-
- @property
- @abstractmethod
- def stimulus_table(self):
- pass
-
- @property
- @abstractmethod
- def snippets_table(self):
- pass
-
-
-class OsDsIndexesTemplateV2(OsDsIndexesTemplate):
- _version = 2
-
- def __init__(self, *args, **kwargs):
- warnings.warn("OsDsIndexesTemplateV2 is deprecated. Use OsDsIndexesTemplate with _version=2 instead.",
- DeprecationWarning, stacklevel=2)
- super().__init__(*args, **kwargs)
-
- @property
- @abstractmethod
- def stimulus_table(self):
- pass
-
- @property
- @abstractmethod
- def snippets_table(self):
- pass
diff --git a/djimaging/tables/response/movingbar/orientation_utils.py b/djimaging/tables/response/movingbar/orientation_utils.py
index 26d2875..3235e49 100644
--- a/djimaging/tables/response/movingbar/orientation_utils.py
+++ b/djimaging/tables/response/movingbar/orientation_utils.py
@@ -1,3 +1,5 @@
+"""Moving-bar selectivity following the Baden et al. 2016 MATLAB implementation."""
+
import cmath
import numpy as np
@@ -92,16 +94,6 @@ def get_dir_idx(snippets, dir_order):
return dir_idx, dir_rad
-def compute_mb_qi(snippets, dir_order):
- assert snippets.ndim == 2
- assert np.asarray(dir_order).ndim == 1
-
- dir_idx, dir_rad = get_dir_idx(snippets, dir_order)
- sorted_responses, sorted_directions = sort_response_matrix(snippets, dir_idx, dir_rad)
- d_qi = quality_index_ds(sorted_responses)
- return d_qi
-
-
def preprocess_mb_snippets(snippets, dir_order):
dir_idx, dir_rad = get_dir_idx(snippets, dir_order)
@@ -135,3 +127,161 @@ def get_si(dir_component, dirs, per):
if per == 2 and direction < 0:
direction += np.pi
return index, direction
+
+
+def get_time_dir_kernels(sorted_responses: np.ndarray, dt: float):
+ """
+ Performs singular value decomposition on the time x direction matrix (averaged across repetitions)
+ Uses a heuristic to try to determine whether a sign flip occurred during svd
+ For the time course, the mean of the first second is subtracted and then the vector is divided by the maximum
+ absolute value.
+ For the direction/orientation tuning curve, the vector is normalized to the range (0,1)
+
+ Parameters:
+ sorted_responses (array): Time x direction matrix.
+ dt (float): 1 / sampling_rate of trace.
+
+ Returns:
+ tuple: Contains time_kernel (array, time x 1), direction_tuning (array, directions x 1), and singular_value (float).
+ """
+ # Only the leading components are used; avoid the full time-by-time U matrix.
+ U, S, Vh = np.linalg.svd(sorted_responses, full_matrices=False)
+
+ time_component = U[:, 0]
+ dir_component = Vh[0, :]
+
+ # the time_kernel determined by SVD should be correlated to the average response across all directions. if the
+ # correlation is negative, U is likely flipped
+
+ if np.mean((-1 * time_component - np.mean(sorted_responses, axis=-1)) ** 2) < np.mean(
+ (time_component - np.mean(sorted_responses, axis=-1)) ** 2
+ ):
+ su = -1
+ else:
+ su = 1
+
+ sv = np.sign(np.mean(np.sign(dir_component)))
+ if sv == 1 and su == 1:
+ s = 1
+ elif sv == -1 and su == -1:
+ s = -1
+ elif sv == 1 and su == -1:
+ s = 1
+ elif sv == 0:
+ s = su
+ else:
+ s = 1
+
+ time_component *= s
+ dir_component *= s
+
+ # determine which entries correspond to the first second, assuming 4 seconds presentation time
+ first_second_idx = np.maximum(int(np.floor(1.0 / dt)), 1)
+ time_component -= np.mean(time_component[:first_second_idx])
+ time_component = time_component / np.max(np.abs(time_component))
+
+ dir_component = math_utils.normalize_zero_one(dir_component)
+
+ return time_component, dir_component
+
+
+def compute_null_dist(dirs: np.ndarray, counts: np.ndarray, per: int, iters=1000):
+ """
+ Test significance of orientation tuning by permutation test.
+
+ Parameters:
+ dirs (array): Vector of directions (#directions x 1) in radians.
+ counts (array): Matrix of responses (#reps x #directions).
+ per (int): Fourier component to test (1 = direction, 2 = orientation).
+ iters (int): Number of permutations for the test.
+
+ Returns:
+ p (float): p-value for tuning.
+ q (float): Magnitude of the Fourier component.
+ qdistr (array): Sampling distribution of |q| under the null hypothesis.
+ """
+ rep_n, dir_n = counts.shape
+ k = dirs.reshape(-1)
+ v = np.exp(per * 1j * k) / np.sqrt(dir_n)
+
+ # Compute magnitude of Fourier component for original data
+ q = np.abs(np.mean(counts, axis=0) @ v)
+
+ # Initialize null distribution
+ qdistr = np.zeros(iters)
+
+ # Flatten counts for permutation
+ flattened_counts = counts.flatten()
+
+ for i in range(iters):
+ # Shuffle counts
+ shuffled_indices = np.random.permutation(rep_n * dir_n)
+ shuffled_counts = flattened_counts[shuffled_indices]
+ shuffled_counts = shuffled_counts.reshape(rep_n, dir_n)
+
+ # Compute Fourier magnitude for shuffled data
+ qdistr[i] = np.abs(np.mean(shuffled_counts, axis=0) @ v)
+
+ # Compute p-value
+ p = np.mean(qdistr > q)
+
+ return p, q, qdistr
+
+
+def compute_os_ds_idxs(snippets: np.ndarray, dir_order: np.ndarray, dt: float, n_shuffles: int = 1000):
+ """
+ Compute orientation selectivity (OS) and direction selectivity (DS) indices.
+
+ Parameters:
+ snippets (array): 2D array of response snippets.
+ dir_order (array): 1D array of direction order.
+ dt (float): Time step.
+ n_shuffles (int): Number of shuffles for null distribution.
+
+ Returns:
+ tuple: Contains various computed indices and components.
+ """
+
+ assert snippets.ndim == 2
+ assert np.asarray(dir_order).ndim == 1
+
+ sorted_directions, sorted_responses, sorted_averages = preprocess_mb_snippets(snippets, dir_order)
+
+ time_component, dir_component = get_time_dir_kernels(sorted_averages, dt=dt)
+
+ dsi, pref_dir = get_si(dir_component, sorted_directions, 1)
+ osi, pref_or = get_si(dir_component, sorted_directions, 2)
+ (t, d, r) = sorted_responses.shape
+ temp = np.reshape(sorted_responses, (t, d * r))
+ projected_flat = temp.T @ time_component # we do this whole projection thing to make the result
+ projected = np.reshape(projected_flat, (d, r)) # between the original and the shuffled comparable
+ surrogate_v = np.mean(projected, axis=-1)
+ surrogate_v -= np.min(surrogate_v)
+ surrogate_v /= np.max(surrogate_v)
+
+ dsi_s, pref_dir_s = get_si(surrogate_v, sorted_directions, 1)
+ # osi_s, pref_or_s = get_si(surrogate_v, sorted_directions, 2) # Not used atm
+
+ p_dsi, _, null_dist_dsi = compute_null_dist(sorted_directions, projected.T, 1, iters=n_shuffles)
+ p_osi, _, null_dist_osi = compute_null_dist(sorted_directions, projected.T, 2, iters=n_shuffles)
+
+ d_qi = quality_index_ds(sorted_responses)
+ on_off = get_on_off_index(time_component, dt=dt)
+
+ return (
+ dsi,
+ p_dsi,
+ null_dist_dsi,
+ pref_dir,
+ osi,
+ p_osi,
+ null_dist_osi,
+ pref_or,
+ on_off,
+ d_qi,
+ time_component,
+ dir_component,
+ surrogate_v,
+ dsi_s,
+ sorted_averages,
+ )
diff --git a/djimaging/tables/response/movingbar/orientation_utils_v1.py b/djimaging/tables/response/movingbar/orientation_utils_v1.py
deleted file mode 100644
index 51561d8..0000000
--- a/djimaging/tables/response/movingbar/orientation_utils_v1.py
+++ /dev/null
@@ -1,116 +0,0 @@
-"""
-This is based on the first python implementation. It introduced some changes compared to the original MATLAB code
-from Baden et al. 2016. This results in fewer p-values < 0.05, but the overall results are similar.
-However, for the classifier this will introduce a bias towards non DS cells.
-"""
-
-import numpy as np
-from scipy import stats
-
-from djimaging.tables.response.movingbar.orientation_utils import quality_index_ds, get_on_off_index, \
- preprocess_mb_snippets, get_si
-from djimaging.utils import math_utils
-
-
-def get_time_dir_kernels(sorted_responses, dt):
- """
- Performs singular value decomposition on the time x direction matrix (averaged across repetitions)
- Uses a heuristic to try to determine whether a sign flip occurred during svd
- For the time course, the mean of the first second is subtracted and then the vector is divided by the maximum
- absolute value.
- For the direction/orientation tuning curve, the vector is normalized to the range (0,1)
- Input:
- sorted_responses: array, time x direction
- dt: 1 / sampling_rate of trace
- Outputs:
- time_kernel array, time x 1 (time component, 1st component of U)
- direction_tuning array, directions x 1 (direction tuning, 1st component of V)
- singular_value float, 1st singular value
- """
-
- U, S, V = np.linalg.svd(sorted_responses)
-
- time_component = U[:, 0]
- dir_component = V[0, :]
-
- # the time_kernel determined by SVD should be correlated to the average response across all directions. if the
- # correlation is negative, U is likely flipped
- r, _ = stats.spearmanr(time_component, np.mean(sorted_responses, axis=-1), axis=1)
- su = np.sign(r)
- if su == 0:
- su = 1
- sv = np.sign(np.mean(np.sign(V[0, :])))
- if sv == 1 and su == 1:
- s = 1
- elif sv == -1 and su == -1:
- s = -1
- elif sv == 1 and su == -1:
- s = 1
- elif sv == 0:
- s = su
- else:
- s = 1
-
- time_component *= s
- dir_component *= s
-
- # determine which entries correspond to the first second, assuming 4 seconds presentation time
- first_second_idx = np.maximum(int(np.round(1. / dt)), 1)
- time_component -= np.mean(time_component[:first_second_idx])
- time_component = time_component / np.max(abs(time_component))
-
- dir_component = math_utils.normalize_zero_one(dir_component)
-
- return time_component, dir_component
-
-
-def compute_null_dist(rep_dir_resps, dirs, per, iters=1000):
- """
- Compute null distribution for direction selectivity
- """
- (rep_n, dir_n) = rep_dir_resps.shape
- flattened = np.reshape(rep_dir_resps, (rep_n * dir_n))
- rand_idx = np.linspace(0, rep_n * dir_n - 1, rep_n * dir_n, dtype=int)
- null_dist = np.zeros(iters)
- for i in range(iters):
- np.random.shuffle(rand_idx)
- shuffled = flattened[rand_idx]
- shuffled = np.reshape(shuffled, (rep_n, dir_n))
- shuffled_mean = np.mean(shuffled, axis=0)
- normalized_shuffled_mean = shuffled_mean - np.min(shuffled_mean)
- normalized_shuffled_mean /= np.max(abs(normalized_shuffled_mean))
- dsi, _ = get_si(normalized_shuffled_mean, dirs, per)
- null_dist[i] = dsi
-
- return null_dist
-
-
-def compute_os_ds_idxs(snippets: np.ndarray, dir_order: np.ndarray, dt: float, n_shuffles: int = 100):
- assert snippets.ndim == 2
- assert np.asarray(dir_order).ndim == 1
-
- sorted_directions, sorted_responses, sorted_averages = preprocess_mb_snippets(snippets, dir_order)
-
- time_component, dir_component = get_time_dir_kernels(sorted_averages, dt=dt)
-
- dsi, pref_dir = get_si(dir_component, sorted_directions, 1)
- osi, pref_or = get_si(dir_component, sorted_directions, 2)
- (t, d, r) = sorted_responses.shape
- temp = np.reshape(sorted_responses, (t, d * r))
- projected = np.dot(np.transpose(temp), time_component) # we do this whole projection thing to make the result
- projected = np.reshape(projected, (d, r)) # between the original and the shuffled comparable
- surrogate_v = np.mean(projected, axis=-1)
- surrogate_v -= np.min(surrogate_v)
- surrogate_v /= np.max(surrogate_v)
-
- dsi_s, pref_dir_s = get_si(surrogate_v, sorted_directions, 1)
- osi_s, pref_or_s = get_si(surrogate_v, sorted_directions, 2)
- null_dist_dsi = compute_null_dist(np.transpose(projected), sorted_directions, 1, iters=n_shuffles)
- p_dsi = np.mean(null_dist_dsi > dsi_s)
- null_dist_osi = compute_null_dist(np.transpose(projected), sorted_directions, 2, iters=n_shuffles)
- p_osi = np.mean(null_dist_osi > osi_s)
- d_qi = quality_index_ds(sorted_responses)
- on_off = get_on_off_index(time_component, dt=dt)
-
- return dsi, p_dsi, null_dist_dsi, pref_dir, osi, p_osi, null_dist_osi, pref_or, \
- on_off, d_qi, time_component, dir_component, surrogate_v, dsi_s, sorted_averages
diff --git a/djimaging/tables/response/movingbar/orientation_utils_v2.py b/djimaging/tables/response/movingbar/orientation_utils_v2.py
deleted file mode 100644
index a6a0d0e..0000000
--- a/djimaging/tables/response/movingbar/orientation_utils_v2.py
+++ /dev/null
@@ -1,166 +0,0 @@
-"""
-This code is very close to the original MATLAB code from Baden et al. 2016.
-"""
-
-import numpy as np
-
-from djimaging.tables.response.movingbar.orientation_utils import quality_index_ds, get_on_off_index, \
- preprocess_mb_snippets, get_si
-from djimaging.utils import math_utils
-
-
-def get_time_dir_kernels(sorted_responses: np.ndarray, dt: float):
- """
- Performs singular value decomposition on the time x direction matrix (averaged across repetitions)
- Uses a heuristic to try to determine whether a sign flip occurred during svd
- For the time course, the mean of the first second is subtracted and then the vector is divided by the maximum
- absolute value.
- For the direction/orientation tuning curve, the vector is normalized to the range (0,1)
-
- Parameters:
- sorted_responses (array): Time x direction matrix.
- dt (float): 1 / sampling_rate of trace.
-
- Returns:
- tuple: Contains time_kernel (array, time x 1), direction_tuning (array, directions x 1), and singular_value (float).
- """
- U, S, Vh = np.linalg.svd(sorted_responses)
-
- time_component = U[:, 0]
- dir_component = Vh[0, :]
-
- # the time_kernel determined by SVD should be correlated to the average response across all directions. if the
- # correlation is negative, U is likely flipped
-
- if np.mean((-1 * time_component - np.mean(sorted_responses, axis=-1)) ** 2) < np.mean(
- (time_component - np.mean(sorted_responses, axis=-1)) ** 2
- ):
- su = -1
- else:
- su = 1
-
- sv = np.sign(np.mean(np.sign(dir_component)))
- if sv == 1 and su == 1:
- s = 1
- elif sv == -1 and su == -1:
- s = -1
- elif sv == 1 and su == -1:
- s = 1
- elif sv == 0:
- s = su
- else:
- s = 1
-
- time_component *= s
- dir_component *= s
-
- # determine which entries correspond to the first second, assuming 4 seconds presentation time
- first_second_idx = np.maximum(int(np.floor(1.0 / dt)), 1)
- time_component -= np.mean(time_component[:first_second_idx])
- time_component = time_component / np.max(np.abs(time_component))
-
- dir_component = math_utils.normalize_zero_one(dir_component)
-
- return time_component, dir_component
-
-
-def compute_null_dist(dirs: np.ndarray, counts: np.ndarray, per: int, iters=1000):
- """
- Test significance of orientation tuning by permutation test.
-
- Parameters:
- dirs (array): Vector of directions (#directions x 1) in radians.
- counts (array): Matrix of responses (#reps x #directions).
- per (int): Fourier component to test (1 = direction, 2 = orientation).
- iters (int): Number of permutations for the test.
-
- Returns:
- p (float): p-value for tuning.
- q (float): Magnitude of the Fourier component.
- qdistr (array): Sampling distribution of |q| under the null hypothesis.
- """
- rep_n, dir_n = counts.shape
- k = dirs.reshape(-1)
- v = np.exp(per * 1j * k) / np.sqrt(dir_n)
-
- # Compute magnitude of Fourier component for original data
- q = np.abs(np.mean(counts, axis=0) @ v)
-
- # Initialize null distribution
- qdistr = np.zeros(iters)
-
- # Flatten counts for permutation
- flattened_counts = counts.flatten()
-
- for i in range(iters):
- # Shuffle counts
- shuffled_indices = np.random.permutation(rep_n * dir_n)
- shuffled_counts = flattened_counts[shuffled_indices]
- shuffled_counts = shuffled_counts.reshape(rep_n, dir_n)
-
- # Compute Fourier magnitude for shuffled data
- qdistr[i] = np.abs(np.mean(shuffled_counts, axis=0) @ v)
-
- # Compute p-value
- p = np.mean(qdistr > q)
-
- return p, q, qdistr
-
-
-def compute_os_ds_idxs(snippets: np.ndarray, dir_order: np.ndarray, dt: float, n_shuffles: int = 1000):
- """
- Compute orientation selectivity (OS) and direction selectivity (DS) indices.
-
- Parameters:
- snippets (array): 2D array of response snippets.
- dir_order (array): 1D array of direction order.
- dt (float): Time step.
- n_shuffles (int): Number of shuffles for null distribution.
-
- Returns:
- tuple: Contains various computed indices and components.
- """
-
- assert snippets.ndim == 2
- assert np.asarray(dir_order).ndim == 1
-
- sorted_directions, sorted_responses, sorted_averages = preprocess_mb_snippets(snippets, dir_order)
-
- time_component, dir_component = get_time_dir_kernels(sorted_averages, dt=dt)
-
- dsi, pref_dir = get_si(dir_component, sorted_directions, 1)
- osi, pref_or = get_si(dir_component, sorted_directions, 2)
- (t, d, r) = sorted_responses.shape
- temp = np.reshape(sorted_responses, (t, d * r))
- projected_flat = temp.T @ time_component # we do this whole projection thing to make the result
- projected = np.reshape(projected_flat, (d, r)) # between the original and the shuffled comparable
- surrogate_v = np.mean(projected, axis=-1)
- surrogate_v -= np.min(surrogate_v)
- surrogate_v /= np.max(surrogate_v)
-
- dsi_s, pref_dir_s = get_si(surrogate_v, sorted_directions, 1)
- # osi_s, pref_or_s = get_si(surrogate_v, sorted_directions, 2) # Not used atm
-
- p_dsi, _, null_dist_dsi = compute_null_dist(sorted_directions, projected.T, 1, iters=n_shuffles)
- p_osi, _, null_dist_osi = compute_null_dist(sorted_directions, projected.T, 2, iters=n_shuffles)
-
- d_qi = quality_index_ds(sorted_responses)
- on_off = get_on_off_index(time_component, dt=dt)
-
- return (
- dsi,
- p_dsi,
- null_dist_dsi,
- pref_dir,
- osi,
- p_osi,
- null_dist_osi,
- pref_or,
- on_off,
- d_qi,
- time_component,
- dir_component,
- surrogate_v,
- dsi_s,
- sorted_averages,
- )
diff --git a/djimaging/tables/response/presentation_quality.py b/djimaging/tables/response/presentation_quality.py
index 19e2bae..6adf946 100644
--- a/djimaging/tables/response/presentation_quality.py
+++ b/djimaging/tables/response/presentation_quality.py
@@ -37,14 +37,14 @@ def definition(self):
# Summarizes Quality Indexes for all ROIs in one presentation
-> self.presentation_table
---
- p_qidx_n_rois : int # Number of ROIs
- p_qidx_median: float # median quality index
- p_qidx_mean: float # mean quality index
- p_qidx_min: float # min quality index
- p_qidx_max: float # max quality index
+ p_qidx_n_rois : int32 # Number of ROIs
+ p_qidx_median: float32 # median quality index
+ p_qidx_mean: float32 # mean quality index
+ p_qidx_min: float32 # min quality index
+ p_qidx_max: float32 # max quality index
'''
for level in self._levels:
- definition += f'p_qidx_f{int(level * 100):03d}: float # fraction of ROIs with qidx >= {level}\n'
+ definition += f'p_qidx_f{int(level * 100):03d}: float32 # fraction of ROIs with qidx >= {level}\n'
return definition
@property
@@ -65,7 +65,7 @@ def key_source(self):
pass
def make(self, key):
- qidxs = (self.qi_table() & key).fetch(self._qidx_col)
+ qidxs = (self.qi_table() & key).to_arrays(self._qidx_col)
new_key = dict(
key,
p_qidx_n_rois=len(qidxs),
diff --git a/djimaging/tables/response/response_quality.py b/djimaging/tables/response/response_quality.py
index 75da168..bc9b05e 100644
--- a/djimaging/tables/response/response_quality.py
+++ b/djimaging/tables/response/response_quality.py
@@ -18,8 +18,8 @@ def definition(self):
# Computes the QI index for responses for repeated stimuli as a signal to noise ratio
-> self.snippets_table
---
- qidx: float # quality index as signal to noise ratio
- min_qidx: float # minimum quality index as 1/r (r = #repetitions)
+ qidx: float32 # quality index as signal to noise ratio
+ min_qidx: float32 # minimum quality index as 1/r (r = #repetitions)
'''
return definition
@@ -72,7 +72,7 @@ def plot(self, restriction=None):
if restriction is None:
restriction = dict()
- qidx, min_qidx = (self & restriction).fetch('qidx', 'min_qidx')
+ qidx, min_qidx = (self & restriction).to_arrays('qidx', 'min_qidx')
fig, axs = plt.subplots(1, 2, figsize=(8, 3))
ax = axs[0]
ax.set(title='qidx')
diff --git a/djimaging/tables/response/sinespot/sinespot_features.py b/djimaging/tables/response/sinespot/sinespot_features.py
index 815840a..1873c95 100644
--- a/djimaging/tables/response/sinespot/sinespot_features.py
+++ b/djimaging/tables/response/sinespot/sinespot_features.py
@@ -18,6 +18,7 @@ class SineSpotFeatures(response.SineSpotFeaturesTemplate):
import numpy as np
from matplotlib import pyplot as plt
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key
from djimaging.utils.trace_utils import get_mean_dt
@@ -31,15 +32,15 @@ def definition(self):
#Computes an OnOff and a transience index based on the chirp step response
-> self.preprocesstraces_table
---
- surround_index : float # large spot response - small spot response
- offset_index : float # strongest small spot response - small spot response
- response_spot_small : float # Mean response to small spot
- response_spot_large : float # Mean response to large spot
- response_spot_a : float # Mean response to small offset spot
- response_spot_b : float # Mean response to small offset spot
- response_spot_c : float # Mean response to small offset spot
- response_spot_d : float # Mean response to small offset spot
- response_rep_x_cond: longblob # Response matrix used to compute suppression
+ surround_index : float32 # large spot response - small spot response
+ offset_index : float32 # strongest small spot response - small spot response
+ response_spot_small : float32 # Mean response to small spot
+ response_spot_large : float32 # Mean response to large spot
+ response_spot_a : float32 # Mean response to small offset spot
+ response_spot_b : float32 # Mean response to small offset spot
+ response_spot_c : float32 # Mean response to small offset spot
+ response_spot_d : float32 # Mean response to small offset spot
+ response_rep_x_cond: # Response matrix used to compute suppression
'''
return definition
@@ -71,7 +72,8 @@ def key_source(self):
def make(self, key):
trace_t0, trace_dt, trace = (self.preprocesstraces_table() & key).fetch1("trace_t0", "trace_dt", "trace")
- triggertimes = (self.presentation_table() & key).fetch1('triggertimes')
+ trace = load_array(trace)
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
ntrigger_rep = (self.stimulus_table() & key).fetch1('ntrigger_rep')
tracetimes = np.arange(len(trace)) * trace_dt + trace_t0
@@ -96,7 +98,7 @@ def plot1(self, key=None, plot_trace=False):
if plot_trace:
(self.preprocesstraces_table & key).plot1()
- response_rep_x_dir = (self & key).fetch1("response_rep_x_cond")
+ response_rep_x_dir = load_array((self & key).fetch1("response_rep_x_cond"))
vabsmax = np.max(np.abs(response_rep_x_dir))
@@ -111,10 +113,10 @@ def plot(self, restriction=None):
if restriction is None:
restriction = dict()
- response_spot_small = (self & restriction).fetch("response_spot_small")
- response_spot_large = (self & restriction).fetch("response_spot_large")
- surround_index = (self & restriction).fetch("surround_index")
- offset_index = (self & restriction).fetch("offset_index")
+ response_spot_small = (self & restriction).to_arrays("response_spot_small")
+ response_spot_large = (self & restriction).to_arrays("response_spot_large")
+ surround_index = (self & restriction).to_arrays("surround_index")
+ offset_index = (self & restriction).to_arrays("offset_index")
fig, axs = plt.subplots(1, 4, figsize=(12, 3))
diff --git a/djimaging/tables/spike_estimation/cascade.py b/djimaging/tables/spike_estimation/cascade.py
index d5ee412..9afc5e6 100644
--- a/djimaging/tables/spike_estimation/cascade.py
+++ b/djimaging/tables/spike_estimation/cascade.py
@@ -38,6 +38,7 @@ class CascadeSpikes(spike_estimation.CascadeSpikesTemplate):
from matplotlib import pyplot as plt
from djimaging.utils import math_utils, plot_utils, trace_utils, filter_utils
+from djimaging.utils.dj_storage import load_array
from djimaging.utils.dj_utils import get_primary_key, suppress_output
@@ -53,14 +54,14 @@ class CascadeTracesParamsTemplate(dj.Lookup):
def definition(self):
definition = """
-> self.stimulus_table
- cas_params_id: int # unique param set id
+ cas_params_id: int32 # unique param set id
---
- window_length: int # window length for SavGol filter in seconds
- poly_order: int # order of polynomial for savgol filter
- q_lower: float # lower percentile for dF/F computation
- q_upper: float # upper percentile for dF/F computation
- f_cutoff: float # cutoff frequency for lowpass filter, only applied when > 0.
- fs_resample = 0 : float # Resampling frequency, only applied when > 0.
+ window_length: int32 # window length for SavGol filter in seconds
+ poly_order: int32 # order of polynomial for savgol filter
+ q_lower: float32 # lower percentile for dF/F computation
+ q_upper: float32 # upper percentile for dF/F computation
+ f_cutoff: float32 # cutoff frequency for lowpass filter, only applied when > 0.
+ fs_resample = 0 : float32 # Resampling frequency, only applied when > 0.
"""
return definition
@@ -76,7 +77,7 @@ def add_default(self, stim_names=None, cas_params_id=1, window_length=60, poly_o
assert 0 <= q_upper <= 100, f'q_upper={q_upper} must be between 0 and 100'
if stim_names is None:
- stim_names = (self.stimulus_table()).fetch('stim_name')
+ stim_names = (self.stimulus_table()).to_arrays('stim_name')
key = dict(
cas_params_id=cas_params_id,
@@ -104,9 +105,9 @@ def definition(self):
-> self.traces_table
-> self.cascadetraces_params_table
---
- pp_trace: longblob # preprocessed trace
- pp_trace_t0: float # start time of trace
- pp_trace_dt: float # time step of trace
+ pp_trace: # preprocessed trace
+ pp_trace_t0: float32 # start time of trace
+ pp_trace_dt: float32 # time step of trace
"""
return definition
@@ -142,7 +143,9 @@ def make(self, key):
warnings.warn("fs_resample is not implemented yet.")
trace_t0, trace_dt, trace = (self.traces_table() & key).fetch1('trace_t0', 'trace_dt', 'trace')
- stim_start = (self.presentation_table() & key).fetch1('triggertimes')[0]
+ trace = load_array(trace)
+ triggertimes = load_array((self.presentation_table() & key).fetch1('triggertimes'))
+ stim_start = triggertimes[0]
if stim_start is not None:
if stim_start < trace_t0:
@@ -159,7 +162,8 @@ def plot1(self, key=None, xlim=None, ylim=None):
pp_trace_t0, pp_trace_dt, pp_trace = (self & key).fetch1("pp_trace_t0", "pp_trace_dt", "pp_trace")
trace_t0, trace_dt, trace = (self.traces_table() & key).fetch1("trace_t0", "trace_dt", "trace")
- triggertimes = (self.presentation_table() & key).fetch1("triggertimes")
+ pp_trace, trace = load_array(pp_trace), load_array(trace)
+ triggertimes = load_array((self.presentation_table() & key).fetch1("triggertimes"))
trace_times = np.arange(len(trace)) * trace_dt + trace_t0
rate_times = np.arange(len(pp_trace)) * pp_trace_dt + pp_trace_t0
@@ -182,7 +186,7 @@ def plot(self, restriction=None, sort=True):
if restriction is None:
restriction = dict()
- cascade_traces = (self & restriction).fetch("pp_trace")
+ cascade_traces = (self & restriction).to_arrays("pp_trace")
cascade_traces = math_utils.padded_vstack(cascade_traces, cval=np.nan)
n = cascade_traces.shape[0]
@@ -203,7 +207,7 @@ class CascadeParamsTemplate(dj.Lookup):
@property
def definition(self):
definition = """
- cascade_params_id: tinyint unsigned # unique param set id
+ cascade_params_id: int32 # unique param set id
---
model_name : varchar(191)
cascade_model_subfolder : varchar(191)
@@ -231,119 +235,105 @@ class CascadeSpikesTemplate(dj.Computed):
@property
def definition(self):
- definition = """
+ return """
# performs basic preprocessing on raw traces
- -> self.cascadetraces_table
+ -> self.cascadetraces_table
-> self.cascade_params_table
---
- spike_prob: longblob
+ spike_prob:
"""
- return definition
@property
@abstractmethod
- def presentation_table(self):
- pass
+ def presentation_table(self): pass
@property
@abstractmethod
- def cascadetraces_params_table(self):
- pass
+ def cascadetraces_params_table(self): pass
@property
@abstractmethod
- def cascadetraces_table(self):
- pass
+ def cascadetraces_table(self): pass
@property
@abstractmethod
- def cascade_params_table(self):
- pass
+ def cascade_params_table(self): pass
@property
def key_source(self):
try:
- return self.presentation_table().proj() * \
- self.cascade_params_table().proj() * \
- self.cascadetraces_params_table.proj()
+ return (self.presentation_table().proj()
+ * self.cascade_params_table().proj()
+ * self.cascadetraces_params_table.proj())
except (AttributeError, TypeError):
- pass
-
- def populate(
- self,
- *restrictions,
- suppress_errors=False,
- return_exception_objects=False,
- reserve_jobs=False,
- order="original",
- limit=None,
- max_calls=None,
- display_progress=False,
- processes=1,
- make_kwargs=None,
- ):
- if len(restrictions) == 0:
- cascade_params_ids = self.cascade_params_table.fetch('cascade_params_id')
- else:
- cascade_params_ids = (self.cascade_params_table & restrictions).fetch('cascade_params_id')
-
- for cascade_params_id in cascade_params_ids:
- cascade_folder, cascade_model_subfolder = (
- self.cascade_params_table & dict(cascade_params_id=cascade_params_id)).fetch1(
- 'cascade_folder', 'cascade_model_subfolder')
-
- if cascade_folder not in sys.path:
- assert os.path.isdir(cascade_folder)
- print(f'Adding {cascade_folder} to sys.path')
- sys.path = [cascade_folder] + sys.path
-
- try:
- from cascade2p import checks
- from cascade2p import cascade
- except ImportError:
- raise ImportError(f'Failed to import cascade from folder {cascade_folder}')
-
- checks.check_packages()
-
- if make_kwargs is None:
- make_kwargs = dict()
- else:
- if 'cascade' in make_kwargs:
- raise ValueError("cascade cannot be passed as make_kwargs.")
- if 'cascade_models_path' in make_kwargs:
- raise ValueError("cascade_models_path cannot be passed as make_kwargs.")
-
- make_kwargs['cascade'] = cascade
- make_kwargs['cascade_models_path'] = os.path.join(cascade_folder, cascade_model_subfolder)
+ return None
+
+ def populate(self, *restrictions, make_kwargs=None, **populate_kwargs):
+ # Validate make_kwargs once
+ make_kwargs = dict(make_kwargs) if make_kwargs else {}
+ for reserved in ('cascade', 'cascade_models_path'):
+ if reserved in make_kwargs:
+ raise ValueError(f"{reserved} cannot be passed as make_kwargs.")
+
+ # Fetch only the columns we need, once
+ params_table = self.cascade_params_table
+ query = params_table & restrictions if restrictions else params_table
+ rows = query.proj('cascade_folder', 'cascade_model_subfolder').to_dicts()
+
+ # Cache imports across params_ids that share a folder
+ cascade_cache = {}
+
+ for row in rows:
+ cascade_folder = row['cascade_folder']
+ cascade_model_subfolder = row['cascade_model_subfolder']
+
+ if cascade_folder not in cascade_cache:
+ if cascade_folder not in sys.path:
+ assert os.path.isdir(cascade_folder)
+ print(f'Adding {cascade_folder} to sys.path')
+ sys.path.insert(0, cascade_folder)
+ try:
+ from cascade2p import checks, cascade
+ except ImportError:
+ raise ImportError(f'Failed to import cascade from folder {cascade_folder}')
+ checks.check_packages()
+ cascade_cache[cascade_folder] = cascade
+
+ cascade = cascade_cache[cascade_folder]
+
+ call_kwargs = {
+ **make_kwargs,
+ 'cascade': cascade,
+ 'cascade_models_path': os.path.join(cascade_folder, cascade_model_subfolder),
+ }
super().populate(
*restrictions,
- dict(cascade_params_id=cascade_params_id),
- suppress_errors=suppress_errors,
- return_exception_objects=return_exception_objects,
- reserve_jobs=reserve_jobs,
- order=order,
- limit=limit,
- max_calls=max_calls,
- display_progress=display_progress,
- processes=processes,
- make_kwargs=make_kwargs,
+ {'cascade_params_id': row['cascade_params_id']},
+ make_kwargs=call_kwargs,
+ **populate_kwargs,
)
def make(self, key, cascade_models_path, cascade, verboselvl=0):
- model_name = (self.cascade_params_table & key).fetch1('model_name')
- pp_traces, roi_ids = (self.cascadetraces_table & key).fetch('pp_trace', 'roi_id')
-
+ pp_traces, roi_ids = (self.cascadetraces_table & key).to_arrays('pp_trace', 'roi_id')
if len(pp_traces) == 0:
return
- elif len(pp_traces) > 1:
- pp_traces = np.stack(pp_traces)
+
+ model_name = (self.cascade_params_table & key).fetch1('model_name')
+ pp_traces = np.stack(pp_traces) if len(pp_traces) > 1 else np.asarray(pp_traces)
with suppress_output(condition=verboselvl == 0):
- spike_probs = cascade.predict(model_name, pp_traces, verbosity=verboselvl, model_folder=cascade_models_path)
+ spike_probs = cascade.predict(
+ model_name, pp_traces,
+ verbosity=verboselvl,
+ model_folder=cascade_models_path,
+ )
- for spike_prob, roi_id in zip(spike_probs, roi_ids):
- self.insert1(dict(key, spike_prob=spike_prob.astype(np.float32), roi_id=roi_id))
+ spike_probs = spike_probs.astype(np.float32, copy=False)
+ self.insert(
+ [{**key, 'spike_prob': sp, 'roi_id': rid}
+ for sp, rid in zip(spike_probs, roi_ids)]
+ )
def plot1(self, key=None, xlim=None):
key = get_primary_key(self, key)
@@ -351,6 +341,7 @@ def plot1(self, key=None, xlim=None):
spike_prob = (self & key).fetch1("spike_prob")
pp_trace_t0, pp_trace_dt, pp_trace = (self.cascadetraces_table & key).fetch1(
"pp_trace_t0", "pp_trace_dt", "pp_trace")
+ spike_prob, pp_trace = load_array(spike_prob), load_array(pp_trace)
trace_times = np.arange(len(pp_trace)) * pp_trace_dt + pp_trace_t0
@@ -367,7 +358,7 @@ def plot(self, restriction=None, sort=True):
if restriction is None:
restriction = dict()
- spike_prob = (self & restriction).fetch("spike_prob")
+ spike_prob = (self & restriction).to_arrays("spike_prob")
spike_prob = math_utils.padded_vstack(spike_prob, cval=np.nan)
n = spike_prob.shape[0]
diff --git a/djimaging/utils/dj_storage.py b/djimaging/utils/dj_storage.py
new file mode 100644
index 0000000..cde463f
--- /dev/null
+++ b/djimaging/utils/dj_storage.py
@@ -0,0 +1,97 @@
+"""Helpers for DataJoint 2 object and array references."""
+
+from __future__ import annotations
+
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import BinaryIO, Iterator, TextIO
+
+import datajoint as dj
+import numpy as np
+
+
+@contextmanager
+def open_object(
+ value: str | Path | dj.ObjectRef,
+ mode: str = "rb",
+ store: str | None = None,
+) -> Iterator[BinaryIO | TextIO]:
+ """Open a local path or a DataJoint ``ObjectRef`` as a file-like object."""
+ if isinstance(value, dj.ObjectRef):
+ with value.open(mode=mode) as file:
+ yield file
+ else:
+ path = file_store_path(value, store) if store is not None else Path(value)
+ with path.open(mode=mode) as file:
+ yield file
+
+
+@contextmanager
+def local_path(
+ value: str | Path | dj.ObjectRef,
+ store: str | None = None,
+) -> Iterator[Path]:
+ """Yield a local path, staging a remote ``ObjectRef`` when necessary."""
+ if not isinstance(value, dj.ObjectRef):
+ yield file_store_path(value, store) if store is not None else Path(value)
+ return
+
+ try:
+ path = Path(value.full_path)
+ except (AttributeError, TypeError, ValueError):
+ path = None
+
+ if path is not None and path.exists():
+ yield path
+ return
+
+ with tempfile.TemporaryDirectory(prefix="djimaging-object-") as directory:
+ destination = Path(directory) / Path(value.path).name
+ yield value.download(destination)
+
+
+def local_file_path(value: str | Path | dj.ObjectRef, store: str | None = None) -> Path:
+ """Return the mounted path for a local-file reference without staging it."""
+ if not isinstance(value, dj.ObjectRef):
+ return file_store_path(value, store) if store is not None else Path(value)
+ try:
+ full_path = value.full_path
+ except (AttributeError, TypeError, ValueError) as error:
+ raise ValueError("This operation requires a locally mounted filepath store") from error
+ if "://" in full_path:
+ raise ValueError("This operation requires a locally mounted filepath store")
+ return Path(full_path)
+
+
+def load_array(value: np.ndarray | dj.NpyRef) -> np.ndarray:
+ """Materialize a DataJoint ``NpyRef`` while leaving arrays unchanged."""
+ return value.load() if isinstance(value, dj.NpyRef) else np.asarray(value)
+
+
+def relative_store_path(value: str | Path, store: str) -> str:
+ """Return a safe path relative to a configured file store."""
+ path = Path(value)
+ store_spec = dj.config.get_store_spec(store)
+ if path.is_absolute():
+ if store_spec.get("protocol") != "file":
+ raise ValueError(f"Absolute paths cannot be inserted into remote store {store!r}")
+ try:
+ path = path.resolve().relative_to(Path(store_spec["location"]).resolve())
+ except ValueError as error:
+ raise ValueError(f"Path {value!r} is outside DataJoint store {store!r}") from error
+ if ".." in path.parts:
+ raise ValueError(f"Path {value!r} escapes DataJoint store {store!r}")
+ return path.as_posix()
+
+
+def file_store_path(value: str | Path, store: str) -> Path:
+ """Resolve a relative value against a local DataJoint file store."""
+ path = Path(value)
+ if path.is_absolute():
+ return path
+
+ store_spec = dj.config.get_store_spec(store)
+ if store_spec.get("protocol") != "file":
+ raise ValueError(f"Store {store!r} is not a local file store")
+ return Path(store_spec["location"]) / path
diff --git a/djimaging/utils/dj_utils.py b/djimaging/utils/dj_utils.py
index 6e79307..7a8cef5 100644
--- a/djimaging/utils/dj_utils.py
+++ b/djimaging/utils/dj_utils.py
@@ -46,9 +46,9 @@ def get_input_tables(definition: str) -> list:
return tables
-def activate_schema(schema: dj.Schema, schema_name: str | None = None,
+def activate_schema(schema: dj.Schema, schema_name: str,
create_schema: bool = True, create_tables: bool = True) -> None:
- """Activate a DataJoint schema, resolving the name from config or parameter.
+ """Activate a DataJoint schema with an explicit schema name.
Based on: https://github.com/datajoint/element-lab
@@ -56,27 +56,16 @@ def activate_schema(schema: dj.Schema, schema_name: str | None = None,
----------
schema : dj.Schema
The DataJoint schema object to activate.
- schema_name : str | None, optional
- Schema name to use. If None, the name is read from ``dj.config['schema_name']``.
+ schema_name : str
+ Schema name to use. DataJoint 2 configuration is strongly typed, so custom
+ settings such as ``dj.config['schema_name']`` are not supported.
create_schema : bool, optional
Whether to create the schema in the database if it does not exist. Default is True.
create_tables : bool, optional
Whether to create tables in the database if they do not exist. Default is True.
- Raises
- ------
- ValueError
- If no schema name can be resolved, or if the provided name conflicts with the
- one stored in ``dj.config``.
"""
- config_schema_name = dj.config.get('schema_name', None)
-
- if not schema_name and not config_schema_name:
- raise ValueError('Must provide schema_name in config or as parameter')
- elif schema_name and config_schema_name and schema_name != config_schema_name:
- raise ValueError('Schema name in config must match schema_name parameter')
-
- schema.activate(schema_name or config_schema_name, create_schema=create_schema, create_tables=create_tables)
+ schema.activate(schema_name, create_schema=create_schema, create_tables=create_tables)
def make_hash(obj: Any) -> str:
@@ -169,29 +158,10 @@ def get_primary_key(table: dj.Table, key: dict | None = None) -> dict:
if key is not None:
key = {k: v for k, v in key.items() if k in table.primary_key}
else:
- key = random.choice(table.proj().fetch(as_dict=True))
+ key = random.choice(table.proj().to_dicts())
return key
-def get_secondary_keys(table: dj.Table) -> list:
- """Get all secondary (non-primary) key names of a DataJoint table.
-
- Parameters
- ----------
- table : dj.Table
- DataJoint table to inspect.
-
- Returns
- -------
- list
- List of attribute name strings that are not part of the primary key.
- """
- key = get_primary_key(table)
- row = (table & key).fetch1()
- secondary_keys = list(set(row.keys()) - set(key.keys()))
- return secondary_keys
-
-
def merge_keys(key1: dict, key2: dict) -> dict:
"""Merge two DataJoint key dictionaries, raising on conflicts.
diff --git a/djimaging/utils/import_helpers.py b/djimaging/utils/import_helpers.py
deleted file mode 100644
index 7b1c134..0000000
--- a/djimaging/utils/import_helpers.py
+++ /dev/null
@@ -1,42 +0,0 @@
-"""Inspired by: sinzlab/nnfabrik: https://github.com/sinzlab/nnfabrik """
-
-import importlib
-
-
-def extract_class_info(full_path: str) -> tuple[str, str]:
- """Separate a fully qualified name into module path and class name.
-
- Parameters
- ----------
- full_path : str
- Fully qualified name, e.g. ``'package.module.ClassName'``.
-
- Returns
- -------
- tuple[str, str]
- ``(module_path, class_name)`` where ``module_path`` is everything
- before the last dot and ``class_name`` is the final component.
- """
- components = full_path.split('.')
- class_identifier = components[-1]
- module_path = '.'.join(components[:-1])
- return module_path, class_identifier
-
-
-def load_class(full_module_path: str, target_class: str) -> type:
- """Dynamically load a class from a module.
-
- Parameters
- ----------
- full_module_path : str
- Dotted import path to the module containing the class.
- target_class : str
- Name of the class attribute to retrieve from the module.
-
- Returns
- -------
- type
- The class object loaded from the specified module.
- """
- loaded_module = importlib.import_module(full_module_path)
- return getattr(loaded_module, target_class)
diff --git a/djimaging/utils/mask_format_utils.py b/djimaging/utils/mask_format_utils.py
index 8e1b26d..7264caa 100644
--- a/djimaging/utils/mask_format_utils.py
+++ b/djimaging/utils/mask_format_utils.py
@@ -22,6 +22,7 @@ def assert_igor_format(roi_mask: np.ndarray) -> None:
If the mask is not 2-D, contains non-integer values, or has unexpected
min/max values.
"""
+ roi_mask = np.asarray(roi_mask)
vmin = np.min(roi_mask)
vmax = np.max(roi_mask)
diff --git a/djimaging/utils/mask_utils.py b/djimaging/utils/mask_utils.py
index 43489a2..71dc63f 100644
--- a/djimaging/utils/mask_utils.py
+++ b/djimaging/utils/mask_utils.py
@@ -12,6 +12,99 @@
from djimaging.utils.scanm import read_h5_utils
from djimaging.utils.cellpose_utils import intersection_over_union
+_ROI_FILE_FORMAT_HELP = (
+ "Use file_format='numpy' for new ROI masks, or 'pickle' to access existing pickle masks."
+)
+
+
+def _infer_roi_file_format(path: str) -> str:
+ """Infer the ROI mask file format from the file extension.
+
+ Parameters
+ ----------
+ path : str
+ Path ending in ``.npy`` or ``.pkl``.
+
+ Returns
+ -------
+ str
+ ``'numpy'`` or ``'pickle'``.
+
+ Raises
+ ------
+ ValueError
+ If the extension is not recognised.
+ """
+ if path.endswith('.npy'):
+ return 'numpy'
+ elif path.endswith('.pkl'):
+ return 'pickle'
+ else:
+ raise ValueError(
+ f"Cannot infer ROI mask format from path={path!r}. "
+ f"Expected a .npy or .pkl file. {_ROI_FILE_FORMAT_HELP}"
+ )
+
+
+def _get_roi_mask_suffix(file_format: str) -> str:
+ if file_format == 'numpy':
+ return '_ROIs.npy'
+ elif file_format == 'pickle':
+ return '_ROIs.pkl'
+ else:
+ raise ValueError(f"Unknown file_format={file_format!r}. {_ROI_FILE_FORMAT_HELP}")
+
+
+def save_roi_mask_file(path: str, roi_mask: np.ndarray, file_format: str) -> None:
+ """Save an ROI mask to disk in either numpy or pickle format.
+
+ Parameters
+ ----------
+ path : str
+ Destination file path. Should end in ``.npy`` for numpy or ``.pkl`` for pickle.
+ roi_mask : np.ndarray
+ The mask array to save.
+ file_format : str
+ ``'numpy'`` to use ``np.save`` or ``'pickle'`` to use ``pickle.dump``.
+ """
+ if file_format == 'numpy':
+ np.save(path, roi_mask)
+ elif file_format == 'pickle':
+ with open(path, 'wb') as f:
+ pickle.dump(roi_mask, f)
+ else:
+ raise ValueError(f"Unknown file_format={file_format!r}. {_ROI_FILE_FORMAT_HELP}")
+
+
+def load_roi_mask_file(path: str) -> np.ndarray:
+ """Load an ROI mask from disk, inferring the format from the file extension.
+
+ Parameters
+ ----------
+ path : str
+ Path to a ``.npy`` (numpy) or ``.pkl`` (pickle) ROI mask file.
+
+ Returns
+ -------
+ np.ndarray
+ The loaded mask array.
+
+ Raises
+ ------
+ ValueError
+ If the file extension is not ``.npy`` or ``.pkl``.
+ """
+ if path.endswith('.npy'):
+ return np.load(path).copy()
+ elif path.endswith('.pkl'):
+ with open(path, 'rb') as f:
+ return pickle.load(f).copy()
+ else:
+ raise ValueError(
+ f"Cannot infer ROI mask format from path={path!r}. "
+ f"Expected a .npy or .pkl file. {_ROI_FILE_FORMAT_HELP}"
+ )
+
def create_circular_mask(h: int, w: int, center: tuple, radius: float) -> np.ndarray:
"""Create a binary circular mask for a 2-D grid.
@@ -653,9 +746,9 @@ def generate_roi_suggestions(mask_pred: np.ndarray, mask_true: np.ndarray, n_art
return rois_to_add
-def to_roi_mask_file(data_file: str, old_suffix: str | None = None, new_suffix: str = '_ROIs.pkl',
+def to_roi_mask_file(data_file: str, old_suffix: str | None = None, new_suffix: str = '_ROIs.npy',
roi_mask_dir: str | None = None, old_prefix: str | None = None,
- new_prefix: str | None = None) -> str:
+ new_prefix: str | None = None, file_format: str | None = None) -> str:
"""Derive the ROI mask file path from a data file path.
Parameters
@@ -666,7 +759,7 @@ def to_roi_mask_file(data_file: str, old_suffix: str | None = None, new_suffix:
Suffix to strip from the filename. If None, the existing file extension
is used.
new_suffix : str, optional
- Suffix to append after stripping `old_suffix`. Default is ``'_ROIs.pkl'``.
+ Suffix to append after stripping `old_suffix`. Default is ``'_ROIs.npy'``.
roi_mask_dir : str | None, optional
Subdirectory name where the mask file should be located. If None, the
same directory as `data_file` is used.
@@ -680,6 +773,9 @@ def to_roi_mask_file(data_file: str, old_suffix: str | None = None, new_suffix:
str
Full path to the corresponding ROI mask file.
"""
+ if file_format is not None:
+ new_suffix = _get_roi_mask_suffix(file_format)
+
f_path, f_name = os.path.split(data_file)
f_root, f_dir = os.path.split(f_path)
@@ -793,10 +889,11 @@ def load_preferred_roi_mask_igor(files: list, mask_alias: str = '',
return None, None
-def load_preferred_roi_mask_pickle(files: list, mask_alias: str = '', highres_alias: str = '',
+def load_preferred_roi_mask_file(files: list, mask_alias: str = '', highres_alias: str = '',
roi_mask_dir: str | None = None, old_prefix: str | None = None,
- new_prefix: str | None = None) -> tuple[np.ndarray | None, str | None]:
- """Load the most preferred ROI mask from a list of files via pickle (Igor format).
+ new_prefix: str | None = None,
+ file_format: str = 'numpy') -> tuple[np.ndarray | None, str | None]:
+ """Load the most preferred ROI mask from a list of files (Igor format).
Parameters
----------
@@ -807,24 +904,38 @@ def load_preferred_roi_mask_pickle(files: list, mask_alias: str = '', highres_al
highres_alias : str, optional
Alias string for high-resolution files (lower preference).
roi_mask_dir : str | None, optional
- Subdirectory override for locating pickle mask files.
+ Subdirectory override for locating mask files.
old_prefix : str | None, optional
Prefix to strip when deriving the mask file name.
new_prefix : str | None, optional
Prefix to prepend when deriving the mask file name.
+ file_format : str | None, optional
+ ``'numpy'`` to look for ``.npy`` files, ``'pickle'`` to look for
+ ``.pkl`` files. Defaults to ``'numpy'``.
Returns
-------
tuple[np.ndarray | None, str | None]
``(roi_mask, filepath)`` in Igor format for the first found mask, or
``(None, None)`` if no mask is found.
+
+ Raises
+ ------
+ ValueError
+ If ``file_format`` is None.
"""
+ if file_format is None:
+ raise ValueError(
+ f"file_format is not set. {_ROI_FILE_FORMAT_HELP} "
+ f"If you were using pickle files before, set file_format='pickle'."
+ )
sorted_files = sort_roi_mask_files(files, mask_alias=mask_alias, highres_alias=highres_alias)
for file in sorted_files:
- roimask_file = to_roi_mask_file(file, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix)
+ roimask_file = to_roi_mask_file(
+ file, roi_mask_dir=roi_mask_dir, old_prefix=old_prefix, new_prefix=new_prefix,
+ file_format=file_format)
if os.path.isfile(roimask_file):
- with open(roimask_file, 'rb') as f:
- roi_mask = pickle.load(f).copy()
+ roi_mask = load_roi_mask_file(roimask_file).astype(np.int32)
roi_mask = to_igor_format(roi_mask)
return roi_mask, roimask_file
else:
diff --git a/djimaging/utils/qdspy/read_logs.py b/djimaging/utils/qdspy/read_logs.py
index 3d5a10c..1a50e58 100644
--- a/djimaging/utils/qdspy/read_logs.py
+++ b/djimaging/utils/qdspy/read_logs.py
@@ -1,8 +1,9 @@
-import json
+import ast
from datetime import datetime
-from pathlib import Path
from typing import List, Dict, Tuple
+_STAGE_INFO_KEYS = frozenset({"scaling_x", "scaling_y", "offset_x", "offset_y", "rotation"})
+
def parse_stimulus_log(log_path: str, verbose: bool = True) -> Tuple[List[Dict], Dict]:
"""
@@ -21,7 +22,7 @@ def parse_stimulus_log(log_path: str, verbose: bool = True) -> Tuple[List[Dict],
Each stimulus entry contains:
- index: Stimulus index
- stimFileName: Name of the stimulus file
- - stimPath: Path to the stimulus file
+ - stimPath: Path to the stimulus file (forward-slash normalised)
- stimMD5: MD5 hash of the stimulus
- t_abs_s: Absolute time from log start (seconds)
- t_since_last_s: Time since last stimulus ended (seconds)
@@ -31,7 +32,10 @@ def parse_stimulus_log(log_path: str, verbose: bool = True) -> Tuple[List[Dict],
- aborted: Whether stimulus was aborted
- t_dur_s_calc: Calculated duration based on frames
- nDroppedFrames: Number of dropped frames
- - params: Additional parameters
+ - params: Additional parameters emitted by the stimulus script
+ - sequenceUsed: Video-sequence index used (MouseCam stimuli only)
+ - stageInfo: Stage calibration dict emitted by newer QDSpy versions
+ (scaling_x/y, offset_x/y, rotation); absent in older logs
"""
nLinesTotal = 0
nLinesData = 0
@@ -42,124 +46,118 @@ def parse_stimulus_log(log_path: str, verbose: bool = True) -> Tuple[List[Dict],
nStims = 0
dt_log_start = None
dt_last_end = None
- iLineLastStart = 0
dt_start = None
stimInfo = None
with open(log_path, "r", encoding="utf-8", errors="ignore") as fLog:
for line in fLog:
- # Extract elements of each line
nLinesTotal += 1
sDateTime = line[:15]
sInfoType = line[15:23].strip().upper()
sMsg = line[23:len(line) - 1].strip()
- # Convert time stamp into a datetime
- dt = datetime.strptime(sDateTime, "%Y%m%d_%H%M%S")
+ try:
+ dt = datetime.strptime(sDateTime, "%Y%m%d_%H%M%S")
+ except ValueError:
+ continue
if nLinesTotal == 1:
- # First line; take as start time
dt_log_start = dt
dt_last_end = dt_log_start
- # Filter for relevant information
if sInfoType not in ["DATA"]:
- # Ignore field
continue
- # Convert data line into dictionary
- sMsg = sMsg.replace("'", "\"")
- sMsg = sMsg.replace("\\\\", "/")
- sMsg = sMsg.replace("(", "[")
- sMsg = sMsg.replace(")", "]")
-
try:
- data = json.loads(sMsg)
- except json.JSONDecodeError as e:
- # JSON parsing failed
+ data = ast.literal_eval(sMsg)
+ except (ValueError, SyntaxError):
if verbose:
print(f"ERROR: parsing line {nLinesTotal - 1} failed:")
print(f"'{sMsg}'")
nLinesErr += 1
- data = None
continue
- # Get stimulus start/stop pairs
- try:
- stimState = data["stimState"].upper()
- except KeyError:
- stimState = None
-
- if stimState:
- # Data contains stimulus information
- if stimState == "STARTED":
- if isStimStarted:
- if verbose:
- print("ERROR: Two consecutive stimulus starts")
- nErr += 1
+ if not isinstance(data, dict):
+ nLinesErr += 1
+ continue
- isStimStarted = True
- iLineLastStart = nLinesData
- dt_start = dt
- t_diff = (dt - dt_log_start).total_seconds()
- t_diff_last = (dt - dt_last_end).total_seconds()
- stimInfo = {
- "index": nStims,
- "stimFileName": Path(data["stimFileName"]).name,
- "stimPath": str(Path(data["stimFileName"]).parent),
- "stimMD5": data["stimMD5"],
- "t_abs_s": t_diff,
- "t_since_last_s": t_diff_last,
- "t_start": dt.time()
- }
-
- elif stimState in ["ABORTED", "FINISHED"]:
- if isStimStarted:
- # Check if stimulus end belongs to stimulus start
- fn = str(Path(stimInfo["stimPath"], stimInfo["stimFileName"])).replace("\\", "/")
- if not data["stimFileName"] == fn:
- if verbose:
- print("ERROR: File paths for stimulus start and end differ")
- nErr += 1
-
- # Append stimulus list entry
- dt_last_end = dt
- t_diff = (dt - dt_start).total_seconds()
- stimInfo.update({
- "aborted": stimState == "ABORTED",
- "t_end": dt.time(),
- "t_dur_s": t_diff
- })
- stims.append(stimInfo)
- nStims += 1
- isStimStarted = False
- else:
+ stimState = data.get("stimState", "").upper()
+
+ if stimState == "STARTED":
+ if isStimStarted:
+ if verbose:
+ print("ERROR: Two consecutive stimulus starts")
+ nErr += 1
+
+ isStimStarted = True
+ dt_start = dt
+ t_diff = (dt - dt_log_start).total_seconds()
+ t_diff_last = (dt - dt_last_end).total_seconds()
+ # Normalise to forward slashes so paths are consistent
+ # regardless of the OS that generated or reads the log.
+ norm_fn = data.get("stimFileName", "").replace("\\", "/")
+ stimInfo = {
+ "index": nStims,
+ "stimFileName": norm_fn.rsplit("/", 1)[-1],
+ "stimPath": norm_fn.rsplit("/", 1)[0] if "/" in norm_fn else "",
+ "stimMD5": data.get("stimMD5", ""),
+ "t_abs_s": t_diff,
+ "t_since_last_s": t_diff_last,
+ "t_start": dt.time(),
+ }
+
+ elif stimState in ["ABORTED", "FINISHED"]:
+ if isStimStarted:
+ norm_fn = data.get("stimFileName", "").replace("\\", "/")
+ fn_start = (
+ stimInfo["stimPath"] + "/" + stimInfo["stimFileName"]
+ if stimInfo["stimPath"]
+ else stimInfo["stimFileName"]
+ )
+ if fn_start != norm_fn:
if verbose:
- print("ERROR: Stimulus end w/o start?")
+ print("ERROR: File paths for stimulus start and end differ")
nErr += 1
- else:
- # Other information
- try:
- _ = data["nFrames"]
- isFrameInfo = True
- except KeyError:
- isFrameInfo = False
-
- if isFrameInfo:
- # Information about stimulus presentation statistics
- if nStims > 0:
- stims[nStims - 1].update({
- "t_dur_s_calc": data["nFrames"] / data["avgFreq_Hz"],
- "nDroppedFrames": data["nDroppedFrames"]
- })
+
+ dt_last_end = dt
+ t_diff = (dt - dt_start).total_seconds()
+ stimInfo.update({
+ "aborted": stimState == "ABORTED",
+ "t_end": dt.time(),
+ "t_dur_s": t_diff,
+ })
+ stims.append(stimInfo)
+ nStims += 1
+ isStimStarted = False
else:
- if nLinesData > iLineLastStart and nLinesData < iLineLastStart + 3:
- # Last start was only up to 2 lines before
- if nStims > 0:
- stims[nStims - 1].update({"params": data})
- else:
- if verbose:
- print("ERROR: Data w/o start??")
- nErr += 1
+ if verbose:
+ print("ERROR: Stimulus end w/o start?")
+ nErr += 1
+
+ elif "nFrames" in data:
+ # Frame statistics arrive after FINISHED, so nStims-1 is correct here.
+ if nStims > 0:
+ stims[nStims - 1].update({
+ "t_dur_s_calc": data["nFrames"] / data["avgFreq_Hz"],
+ "nDroppedFrames": data["nDroppedFrames"],
+ })
+
+ elif isStimStarted:
+ # Extra DATA lines emitted while a stimulus is running.
+ # Classify by content rather than position to support all QDSpy versions.
+ if "SequenceUsed" in data:
+ stimInfo["sequenceUsed"] = data["SequenceUsed"]
+ elif set(data.keys()) == _STAGE_INFO_KEYS:
+ # Stage calibration line added in newer QDSpy versions.
+ stimInfo["stageInfo"] = data
+ else:
+ # Treat as stimulus parameters; last assignment wins if emitted
+ # multiple times (older QDSpy versions emitted params after stage info).
+ stimInfo["params"] = data
+
+ else:
+ if verbose:
+ print("ERROR: Data w/o start??")
+ nErr += 1
nLinesData += 1
@@ -171,7 +169,7 @@ def parse_stimulus_log(log_path: str, verbose: bool = True) -> Tuple[List[Dict],
"nLinesTotal": nLinesTotal,
"nLinesData": nLinesData,
"nLinesErr": nLinesErr,
- "nErr": nErr
+ "nErr": nErr,
}
return stims, stats
diff --git a/djimaging/utils/receptive_fields/fit_rf_utils.py b/djimaging/utils/receptive_fields/fit_rf_utils.py
index 9d64e23..fe7e225 100644
--- a/djimaging/utils/receptive_fields/fit_rf_utils.py
+++ b/djimaging/utils/receptive_fields/fit_rf_utils.py
@@ -20,298 +20,6 @@
import numpy as np
from numba import jit
-from tqdm.notebook import tqdm
-
-
-def compute_linear_rf(dt: float, trace: np.ndarray, stim: np.ndarray,
- frac_train: float, frac_dev: float,
- filter_dur_s_past: float, filter_dur_s_future: float = 0.,
- kind: str = 'sta', threshold_pred: bool = False,
- dtype: type = np.float32,
- batch_size_n: int = 6_000_000,
- verbose: bool = False) -> tuple:
- """Compute a linear receptive field (STA or MLE) from stimulus and response.
-
- Parameters
- ----------
- dt : float
- Time step in seconds.
- trace : np.ndarray
- Neural response trace, shape (n_frames,).
- stim : np.ndarray
- Stimulus array, shape (n_frames, ...).
- frac_train : float
- Fraction of data used for training.
- frac_dev : float
- Fraction of data used for development/validation.
- filter_dur_s_past : float
- Duration of the filter in the past (seconds).
- filter_dur_s_future : float, optional
- Duration of the filter in the future (seconds). Default is 0.
- kind : str, optional
- Estimation method: 'sta' or 'mle'. Default is 'sta'.
- threshold_pred : bool, optional
- Whether to clip predictions at zero. Default is False.
- dtype : type, optional
- Data type for computations. Default is np.float32.
- batch_size_n : int, optional
- Maximum number of elements before switching to batch mode. Default is 6_000_000.
- verbose : bool, optional
- Whether to print progress. Default is False.
-
- Returns
- -------
- tuple
- rf : np.ndarray
- Estimated receptive field reshaped to (dim_t, ...).
- rf_time : np.ndarray
- Time axis for the RF.
- model_eval : dict
- Evaluation metrics including predictions and correlation coefficients.
- x : dict
- Split stimulus data.
- y : dict
- Split trace data.
- shift : int
- Temporal shift applied during design matrix construction.
- """
- kind = kind.lower()
-
- assert trace.size == stim.shape[0]
- assert kind in ['sta', 'mle'], f"kind={kind}"
-
- rf_time, dim_t, shift, burn_in = get_rf_timing_params(filter_dur_s_past, filter_dur_s_future, dt)
- dims = (dim_t,) + stim.shape[1:]
-
- x, y = split_data(x=stim, y=trace, frac_train=frac_train, frac_dev=frac_dev, as_dict=True)
-
- rf, y_pred_train = compute_linear_rf_single_or_batch(
- x_train=x['train'], y_train=y['train'], dim_t=dim_t, shift=shift, burn_in=burn_in, kind=kind,
- threshold_pred=threshold_pred, batch_size_n=batch_size_n, stim_shape=stim.shape, dtype=dtype, verbose=verbose)
-
- model_eval = dict()
- model_eval['burn_in'] = burn_in
- model_eval['y_pred_train'] = y_pred_train
- model_eval['cc_train'] = np.corrcoef(y['train'][burn_in:], y_pred_train)[0, 1]
- model_eval['mse_train'] = np.mean((y['train'][burn_in:] - y_pred_train) ** 2)
-
- if 'test' in x:
- x_test_dm = build_design_matrix(x['test'], n_lag=dim_t, shift=shift, dtype=dtype)[burn_in:]
- y_pred_test = predict_linear_rf_response(
- rf=rf, stim_design_matrix=x_test_dm, threshold=threshold_pred, dtype=dtype)
-
- model_eval['y_pred_test'] = y_pred_test
- model_eval['cc_test'] = np.corrcoef(y['test'][burn_in:], y_pred_test)[0, 1]
- model_eval['mse_test'] = np.mean((y['test'][burn_in:] - y_pred_test) ** 2)
-
- rf = rf.reshape(dims)
- return rf, rf_time, model_eval, x, y, shift
-
-
-def compute_linear_rf_single_or_batch(x_train: np.ndarray, y_train: np.ndarray,
- dim_t: int, shift: int, burn_in: int,
- kind: str, threshold_pred: bool,
- stim_shape: tuple,
- batch_size_n: int = 6_000_000,
- is_design_matrix: bool = False,
- dtype: type = np.float32,
- verbose: bool = False) -> tuple[np.ndarray, np.ndarray]:
- """Compute linear RF either in one pass or in batches depending on data size.
-
- Parameters
- ----------
- x_train : np.ndarray
- Training stimulus data.
- y_train : np.ndarray
- Training neural response data.
- dim_t : int
- Number of temporal filter taps.
- shift : int
- Temporal shift for the design matrix.
- burn_in : int
- Number of initial frames to skip.
- kind : str
- Estimation method: 'sta' or 'mle'.
- threshold_pred : bool
- Whether to clip predictions at zero.
- stim_shape : tuple
- Original shape of the stimulus array.
- batch_size_n : int, optional
- Maximum number of elements before switching to batch mode. Default is 6_000_000.
- is_design_matrix : bool, optional
- Whether x_train is already a design matrix. Default is False.
- dtype : type, optional
- Data type for computations. Default is np.float32.
- verbose : bool, optional
- Whether to print progress. Default is False.
-
- Returns
- -------
- tuple[np.ndarray, np.ndarray]
- rf : np.ndarray
- Estimated receptive field.
- y_pred_train : np.ndarray
- Predicted training responses.
- """
- if np.product(stim_shape) <= batch_size_n:
- rf, y_pred_train = _compute_linear_rf_single_batch(
- x_train=x_train, y_train=y_train, kind=kind, dim_t=dim_t, shift=shift, burn_in=burn_in,
- is_design_matrix=is_design_matrix, threshold_pred=threshold_pred, dtype=dtype)
- else:
- assert kind == 'sta', f"kind={kind} not supported for compute_per='feature'"
- rf, y_pred_train = _compute_sta_batchwise(
- x_train=x_train, y_train=y_train, kind=kind, dim_t=dim_t, shift=shift, burn_in=burn_in,
- threshold_pred=threshold_pred, dtype=dtype,
- is_design_matrix=is_design_matrix, batch_size=np.maximum(batch_size_n // stim_shape[0], 1), verbose=verbose)
-
- return rf, y_pred_train
-
-
-def _compute_linear_rf_single_batch(x_train: np.ndarray, y_train: np.ndarray,
- dim_t: int, shift: int, burn_in: int,
- kind: str, threshold_pred: bool,
- is_design_matrix: bool = False,
- dtype: type = np.float32) -> tuple[np.ndarray, np.ndarray]:
- """Compute linear RF from a single batch of training data.
-
- Parameters
- ----------
- x_train : np.ndarray
- Training stimulus data or design matrix.
- y_train : np.ndarray
- Training neural response data.
- dim_t : int
- Number of temporal filter taps.
- shift : int
- Temporal shift for the design matrix.
- burn_in : int
- Number of initial frames to skip.
- kind : str
- Estimation method: 'sta' or 'mle'.
- threshold_pred : bool
- Whether to clip predictions at zero.
- is_design_matrix : bool, optional
- Whether x_train is already a design matrix. Default is False.
- dtype : type, optional
- Data type for computations. Default is np.float32.
-
- Returns
- -------
- tuple[np.ndarray, np.ndarray]
- rf : np.ndarray
- Estimated receptive field.
- y_pred_train : np.ndarray
- Predicted training responses.
- """
- if is_design_matrix:
- x_train_dm = x_train[burn_in:].astype(dtype)
- else:
- x_train_dm = build_design_matrix(x_train, n_lag=dim_t, shift=shift, dtype=dtype)[burn_in:]
-
- y_train_dm = y_train[burn_in:].astype(dtype)
-
- if kind == 'sta':
- rf = compute_rf_sta(X=x_train_dm, y=y_train_dm)
- elif kind == 'mle':
- rf = compute_rf_mle(X=x_train_dm, y=y_train_dm)
- else:
- raise NotImplementedError(kind)
-
- y_pred_train = predict_linear_rf_response(
- rf=rf, stim_design_matrix=x_train_dm, threshold=threshold_pred, dtype=dtype)
-
- return rf, y_pred_train
-
-
-def _compute_sta_batchwise(x_train: np.ndarray, y_train: np.ndarray,
- dim_t: int, shift: int, burn_in: int,
- kind: str, threshold_pred: bool,
- batch_size: int = 400,
- is_design_matrix: bool = False,
- dtype: type = np.float32,
- verbose: bool = False) -> tuple[np.ndarray, np.ndarray]:
- """Compute STA receptive field in batches over feature dimensions.
-
- Parameters
- ----------
- x_train : np.ndarray
- Training stimulus data, shape (n_frames, ...).
- y_train : np.ndarray
- Training neural response data, shape (n_frames,).
- dim_t : int
- Number of temporal filter taps.
- shift : int
- Temporal shift for the design matrix.
- burn_in : int
- Number of initial frames to skip.
- kind : str
- Estimation method: 'sta' or 'mle'.
- threshold_pred : bool
- Whether to clip predictions at zero.
- batch_size : int, optional
- Number of features per batch. Default is 400.
- is_design_matrix : bool, optional
- Whether x_train is already a design matrix. Default is False.
- dtype : type, optional
- Data type for computations. Default is np.float32.
- verbose : bool, optional
- Whether to display a progress bar. Default is False.
-
- Returns
- -------
- tuple[np.ndarray, np.ndarray]
- rf : np.ndarray
- Estimated receptive field, shape (dim_t, ...).
- y_pred_train : np.ndarray
- Summed predicted training responses across batches.
- """
- x_shape = x_train.shape
- n_frames = x_shape[0]
- n_features = np.product(x_shape[1:])
-
- rf = np.zeros((dim_t, n_features), dtype=dtype)
-
- x_train = x_train.reshape((n_frames, n_features))
- y_train_dm = y_train[burn_in:].astype(dtype)
-
- batches = np.array_split(np.arange(n_features), np.ceil(n_features / batch_size))
- y_pred_train_per_batch = np.zeros((n_frames - burn_in, len(batches)), dtype=dtype)
-
- bar = tqdm(batches, desc='STA batches', leave=False) if verbose else None
-
- for i, batch in enumerate(batches):
- if is_design_matrix:
- x_train_dm_i = x_train[burn_in:, batch]
- else:
- x_train_dm_i = build_design_matrix(x_train[:, batch], n_lag=dim_t, shift=shift, dtype=dtype)[burn_in:]
-
- if kind == 'sta':
- rf_i = compute_rf_sta(X=x_train_dm_i, y=y_train_dm)
- elif kind == 'mle':
- rf_i = compute_rf_mle(X=x_train_dm_i, y=y_train_dm)
- else:
- raise NotImplementedError()
-
- rf[:, batch] = rf_i.reshape(-1, len(batch))
- y_pred_train_per_batch[:, i] = predict_linear_rf_response(
- rf=rf_i, stim_design_matrix=x_train_dm_i, threshold=False, dtype=dtype)
-
- if bar is not None:
- bar.update(1)
-
- if bar is not None:
- bar.close()
-
- # Reshape back. Is this necessary for x?
- rf = rf.reshape((dim_t,) + x_shape[1:])
- x_train.reshape(x_shape)
-
- y_pred_train = np.sum(y_pred_train_per_batch, axis=1)
-
- if threshold_pred:
- y_pred_train = np.clip(y_pred_train, 0, None)
-
- return rf, y_pred_train
def get_rf_timing_params(filter_dur_s_past: float,
@@ -369,57 +77,6 @@ def compute_rf_sta(X: np.ndarray, y: np.ndarray) -> np.ndarray:
return w
-@jit(nopython=True)
-def compute_rf_mle(X: np.ndarray, y: np.ndarray) -> np.ndarray:
- """Compute maximum likelihood estimate (MLE) receptive field. From RFEst.
-
- Parameters
- ----------
- X : np.ndarray
- Design matrix, shape (n_samples, n_features).
- y : np.ndarray
- Neural response vector, shape (n_samples,).
-
- Returns
- -------
- np.ndarray
- MLE receptive field, shape (n_features,).
- """
- XtY = X.T @ y
- XtX = X.T @ X
- w = np.linalg.lstsq(XtX, XtY)[0]
- return w
-
-
-def predict_linear_rf_response(rf: np.ndarray, stim_design_matrix: np.ndarray,
- threshold: bool = False,
- dtype: type = np.float32) -> np.ndarray:
- """Predict neural response using a linear receptive field and design matrix.
-
- Parameters
- ----------
- rf : np.ndarray
- Receptive field weights, shape (n_features,).
- stim_design_matrix : np.ndarray
- Stimulus design matrix, shape (n_samples, n_features).
- threshold : bool, optional
- Whether to clip negative predictions to zero. Default is False.
- dtype : type, optional
- Output data type. Default is np.float32.
-
- Returns
- -------
- np.ndarray
- Predicted neural response, shape (n_samples,).
- """
- y_pred = stim_design_matrix @ rf
-
- if threshold:
- y_pred = np.clip(y_pred, 0, None)
-
- return y_pred.astype(dtype)
-
-
def split_data(x: np.ndarray, y: np.ndarray,
frac_train: float = 0.8, frac_dev: float = 0.1,
as_dict: bool = False):
@@ -480,35 +137,35 @@ def split_data(x: np.ndarray, y: np.ndarray,
def build_design_matrix(X: np.ndarray, n_lag: int, shift: int = 0,
- n_c: int = 1, dtype: type = None) -> np.ndarray:
+ dtype: type = None) -> np.ndarray:
"""Build design matrix for linear RF estimation.
- Modified from RFEst.
+ Modified from RFEst. Works for multi-color stimuli by flattening all
+ non-temporal dimensions (spatial and color) into a single feature axis.
Parameters
----------
X : np.ndarray
- Stimulus array, shape (n_frames, ...).
+ Stimulus array, shape (n_frames, ...). For multi-color stimuli this
+ can be e.g. (n_frames, n_x, n_y, n_colors).
n_lag : int
Number of temporal lags to include.
shift : int, optional
Temporal shift applied to the design matrix (negative shifts into future). Default is 0.
- n_c : int, optional
- Number of color channels (adds an extra dimension if > 1). Default is 1.
dtype : type, optional
Output data type. If None, uses X.dtype.
Returns
-------
np.ndarray
- Design matrix, shape (n_frames, n_lag * n_feature) or
- (n_frames, n_lag * n_feature / n_c, n_c) if n_c > 1.
+ Design matrix, shape (n_frames, n_lag * n_feature) where
+ n_feature = product of all non-temporal dimensions.
"""
if dtype is None:
dtype = X.dtype
n_frames = X.shape[0]
- n_feature = np.product(X.shape[1:])
+ n_feature = np.prod(X.shape[1:])
X_design = np.reshape(X.copy(), (n_frames, n_feature))
@@ -520,7 +177,4 @@ def build_design_matrix(X: np.ndarray, n_lag: int, shift: int = 0,
X_design = np.hstack([X_design[i:n_frames + i] for i in range(n_lag)])
- if n_c > 1:
- X_design = np.reshape(X_design, (X_design.shape[0], -1, n_c))
-
return X_design.astype(dtype)
diff --git a/djimaging/utils/receptive_fields/glm_utils.py b/djimaging/utils/receptive_fields/glm_utils.py
index 3b8cd0f..6640139 100644
--- a/djimaging/utils/receptive_fields/glm_utils.py
+++ b/djimaging/utils/receptive_fields/glm_utils.py
@@ -34,15 +34,24 @@ def __init__(self, dt: float, stim: np.ndarray, trace: np.ndarray,
fit_R: bool = False, fit_intercept: bool = True,
num_subunits: int = 1, distr: str = 'gaussian',
frac_test: float = 0.2, t_burn_min: float = 0.,
- shift: int = None):
+ shift: int = None, channel_names=None):
"""Initialize a GLM-based receptive field estimator.
+ Supports single-channel and multi-channel (e.g. two-color) stimuli.
+ For multi-channel data, one filter per channel is fit and the
+ contributions are summed before the output nonlinearity:
+ y = nonlin( sum_c X_c @ w_c + intercept )
+
Parameters
----------
dt : float
Time step in seconds.
stim : np.ndarray
- Stimulus array, shape (n_frames,) or (n_frames, n_y, n_x).
+ Stimulus array. Accepted shapes:
+ - (n_frames,) 1D flicker, single channel
+ - (n_frames, n_colors) 1D flicker, multi-channel
+ - (n_frames, n_y, n_x) spatial, single channel
+ - (n_frames, n_y, n_x, n_colors) spatial, multi-channel
trace : np.ndarray
Neural response trace, shape (n_frames,).
filter_dur_s_past : float
@@ -99,6 +108,9 @@ def __init__(self, dt: float, stim: np.ndarray, trace: np.ndarray,
Minimum burn-in time in seconds. Default is 0.
shift : int, optional
Expected temporal shift; if provided, validated against computed value.
+ channel_names : list of str, optional
+ Names for each color/stimulus channel. If None, defaults to
+ ['stimulus'] for single-channel or ['ch0', 'ch1', ...] for multi-channel.
"""
# Data
self.dt = dt
@@ -146,29 +158,115 @@ def __init__(self, dt: float, stim: np.ndarray, trace: np.ndarray,
self.burn_in = np.maximum(int(np.ceil(t_burn_min / dt)), burn_in)
- if stim.ndim == 3:
+ # Detect stimulus shape: how many channels, and what spatial dims
+ # Shapes supported:
+ # (T,) -> 1D flicker, 1 channel
+ # (T, C) -> 1D flicker, C channels
+ # (T, Y, X) -> spatial, 1 channel
+ # (T, Y, X, C) -> spatial, C channels
+ if stim.ndim == 4:
+ self.n_colors = stim.shape[3]
self.hdims = stim.shape[1]
self.wdims = stim.shape[2]
self.dims = (self.dim_t, self.hdims, self.wdims)
+ self.is_spatial = True
+ elif stim.ndim == 3:
+ self.n_colors = 1
+ self.hdims = stim.shape[1]
+ self.wdims = stim.shape[2]
+ self.dims = (self.dim_t, self.hdims, self.wdims)
+ self.is_spatial = True
+ elif stim.ndim == 2:
+ self.n_colors = stim.shape[1]
+ self.hdims = None
+ self.wdims = None
+ self.dims = (self.dim_t,)
+ self.is_spatial = False
elif stim.ndim == 1:
+ self.n_colors = 1
+ self.hdims = None
+ self.wdims = None
self.dims = (self.dim_t,)
+ self.is_spatial = False
else:
- raise NotImplementedError()
+ raise NotImplementedError(f"{stim.ndim=}")
+
+ # Channel naming: keep 'stimulus' as the single-channel name for
+ # backward compatibility with downstream code that reads model.w['opt']['stimulus'].
+ if channel_names is None:
+ if self.n_colors == 1:
+ self.channel_names = ['stimulus']
+ else:
+ self.channel_names = [f'ch{i}' for i in range(self.n_colors)]
+ else:
+ assert len(channel_names) == self.n_colors, \
+ f"channel_names has length {len(channel_names)} but stim has {self.n_colors} channels"
+ self.channel_names = list(channel_names)
self.df_ts = self.clip_dfs(np.sort(np.asarray(df_ts)), self.dim_t)
- if stim.ndim == 3:
+ if self.is_spatial:
self.df_ws = self.clip_dfs(np.sort(np.asarray(df_ws)), self.wdims)
else:
assert not df_ws
- def compute_glm_receptive_field(self) -> tuple[np.ndarray, dict, dict]:
+ def _get_channel_stim(self, stim: np.ndarray, c: int) -> np.ndarray:
+ """Extract the c-th channel from a (possibly multi-channel) stimulus array.
+
+ Parameters
+ ----------
+ stim : np.ndarray
+ Stimulus array in any of the supported shapes.
+ c : int
+ Channel index.
+
+ Returns
+ -------
+ np.ndarray
+ Single-channel stimulus, shape (T,) or (T, Y, X).
+ """
+ if self.n_colors == 1:
+ return stim
+ if self.is_spatial:
+ # (T, Y, X, C) -> (T, Y, X)
+ return stim[..., c]
+ else:
+ # (T, C) -> (T,)
+ return stim[:, c]
+
+ def _build_design_matrices(self, stim: np.ndarray) -> dict:
+ """Build a per-channel dict of design matrices.
+
+ Parameters
+ ----------
+ stim : np.ndarray
+ Stimulus array (may include channel dim).
+
+ Returns
+ -------
+ dict
+ Mapping channel_name -> design matrix (np.ndarray).
+ """
+ Xs = {}
+ for c, name in enumerate(self.channel_names):
+ stim_c = self._get_channel_stim(stim, c)
+ Xs[name] = build_design_matrix(
+ X=stim_c, n_lag=self.dims[0], shift=self.shift, dtype=np.float32)
+ return Xs
+
+ def _slice_design_matrices(self, Xs: dict, idx) -> dict:
+ """Slice every per-channel design matrix along axis 0 with the same index."""
+ return {name: X[idx] for name, X in Xs.items()}
+
+ def compute_glm_receptive_field(self) -> tuple:
"""Estimate RF for given data. Return RF and quality.
Returns
-------
tuple[np.ndarray, dict, dict]
w : np.ndarray
- Estimated receptive field.
+ Estimated receptive field. For single-channel: shape == self.dims.
+ For multi-channel: shape == (n_colors,) + self.dims, with channels
+ ordered according to self.channel_names.
quality_dict : dict
Quality metrics including correlation coefficients and permutation test results.
model_dict : dict
@@ -187,10 +285,28 @@ def compute_glm_receptive_field(self) -> tuple[np.ndarray, dict, dict]:
if self.verbose > 0:
print("############ Evaluate model performance ############")
+ # Build per-channel design matrices for test data (used by perm test + predict)
+ if self.frac_test > 0:
+ Xs_test = self._build_design_matrices(stim_dict['test'])
+ else:
+ Xs_test = None
+
if (self.n_perm is not None) and (self.n_perm > 0) and (self.frac_test > 0):
assert trace_dict['test'].size > 1, trace_dict['test']
- score_trueX, score_permX = rfest.check.compute_permutation_test(
- model, stim_dict['test'], trace_dict['test'], n_perm=self.n_perm, metric=self.metric, history=False)
+ # rfest.check.compute_permutation_test expects either a single stim array
+ # (single-channel case) or a dict of per-channel design matrices.
+ if self.n_colors == 1:
+ perm_input = stim_dict['test']
+ score_trueX, score_permX = rfest.check.compute_permutation_test(
+ model, perm_input, trace_dict['test'],
+ n_perm=self.n_perm, metric=self.metric, history=False)
+ else:
+ # Manual permutation test for multi-channel: jointly shuffle
+ # all channels in time so cross-channel correlations are preserved
+ # under the null but the stim<->trace mapping is broken.
+ score_trueX, score_permX = self._compute_permutation_test_multichannel(
+ model, Xs_test, trace_dict['test'], n_perm=self.n_perm, metric=self.metric)
+
p_value = quality_test(
score_trueX=score_trueX, score_permX=score_permX, metric=self.metric)
quality_dict = dict(
@@ -199,24 +315,33 @@ def compute_glm_receptive_field(self) -> tuple[np.ndarray, dict, dict]:
if self.kfold != 1:
y_pred_train = model.forwardpass(kind='train', p=model.p['opt'])
else:
- y_pred_train = model.predict({'stimulus': stim_dict['train']})
+ # For prediction, build per-channel design matrices from training stim.
+ Xs_train_pred = self._build_design_matrices(stim_dict['train'])
+ y_pred_train = model.predict(Xs_train_pred)
for metric_ in ['corrcoef', 'mse']:
quality_dict[f'{metric_}_train'] = model.compute_score(
y=trace_dict['train'][model.burn_in:], y_pred=y_pred_train, metric=metric_)
if self.frac_test > 0:
- y_test_pred = model.predict({'stimulus': stim_dict['test']})
+ y_test_pred = model.predict(Xs_test)
quality_dict[f'{metric_}_test'] = model.compute_score(
y=trace_dict['test'][model.burn_in:], y_pred=y_test_pred, metric=metric_)
else:
y_test_pred = None
- w = rfest.utils.uvec(model.w['opt']['stimulus'])
- if w.shape[1] != 1:
- raise NotImplementedError('Not implemented for more than one subunit')
+ # Extract per-channel RFs.
+ ws = {}
+ for name in self.channel_names:
+ w_c = rfest.utils.uvec(model.w['opt'][name])
+ if w_c.shape[1] != 1:
+ raise NotImplementedError('Not implemented for more than one subunit')
+ ws[name] = np.array(w_c.reshape(model.dims[name]))
- w = np.array(w.reshape(model.dims['stimulus']))
+ if self.n_colors == 1:
+ w = ws[self.channel_names[0]]
+ else:
+ w = np.stack([ws[name] for name in self.channel_names], axis=0)
model_dict = dict(
rf_time=self.rf_time, dt=self.dt,
@@ -233,6 +358,9 @@ def compute_glm_receptive_field(self) -> tuple[np.ndarray, dict, dict]:
train_stop=model.train_stop,
y_train=np.array(trace_dict['train'][model.burn_in:]).astype(np.float32),
y_pred_train=np.array(y_pred_train).astype(np.float32),
+ channel_names=list(self.channel_names),
+ n_colors=self.n_colors,
+ ws_per_channel={name: ws[name] for name in self.channel_names},
)
if self.frac_test > 0:
@@ -261,6 +389,44 @@ def compute_glm_receptive_field(self) -> tuple[np.ndarray, dict, dict]:
return w, quality_dict, model_dict
+ def _compute_permutation_test_multichannel(
+ self, model, Xs_test: dict, y_test: np.ndarray,
+ n_perm: int, metric: str) -> tuple:
+ """Permutation test for multi-channel models.
+
+ Permutes y_test (rather than the stimulus) so cross-channel correlation
+ structure is preserved under the null. Equivalent to jointly shuffling
+ all channels in time.
+
+ Parameters
+ ----------
+ model : object
+ Fitted GLM.
+ Xs_test : dict
+ Per-channel test design matrices.
+ y_test : np.ndarray
+ Test response trace.
+ n_perm : int
+ Number of permutations.
+ metric : str
+ Score metric ('corrcoef' or 'mse').
+
+ Returns
+ -------
+ tuple
+ (score_true, score_perm_array)
+ """
+ y_pred = model.predict(Xs_test)
+ y_true = y_test[model.burn_in:]
+ score_true = float(model.compute_score(y=y_true, y_pred=y_pred, metric=metric))
+
+ rng = np.random.RandomState(self.seed)
+ score_perm = np.zeros(n_perm, dtype=np.float32)
+ for i in range(n_perm):
+ y_shuf = rng.permutation(y_true)
+ score_perm[i] = float(model.compute_score(y=y_shuf, y_pred=y_pred, metric=metric))
+ return score_true, score_perm
+
def dfh_from_dfw(self, df_w: int) -> int:
"""Compute height degrees of freedom from width degrees of freedom.
@@ -284,7 +450,7 @@ def compute_best_rf_model(self, trace: np.ndarray, stim: np.ndarray) -> tuple:
trace : np.ndarray
Neural response trace.
stim : np.ndarray
- Stimulus array.
+ Stimulus array (may include channel dim).
Returns
-------
@@ -296,41 +462,57 @@ def compute_best_rf_model(self, trace: np.ndarray, stim: np.ndarray) -> tuple:
"""
if self.verbose > 0 and (self.betas.size > 1):
+ df_ws_size = self.df_ws.size if self.is_spatial else 1
print(f"################## Optimizing dfs ##################\n" + \
- f"\tTrying {self.df_ts.size * self.df_ws.size} combination:" + \
- f" df_ts={self.df_ts} and df_ws={self.df_ws}")
+ f"\tTrying {self.df_ts.size * df_ws_size} combination:" + \
+ f" df_ts={self.df_ts} and df_ws={self.df_ws if self.is_spatial else None}")
# Define HP grid
- dfs = [(df_t, self.dfh_from_dfw(df_w), df_w) for df_t, df_w in list(itertools.product(self.df_ts, self.df_ws))]
+ if self.is_spatial:
+ dfs = [(df_t, self.dfh_from_dfw(df_w), df_w)
+ for df_t, df_w in list(itertools.product(self.df_ts, self.df_ws))]
+ else:
+ dfs = [(df_t,) for df_t in self.df_ts]
+
metrics_dev_opt = np.zeros((np.maximum(self.kfold, 1), len(dfs), len(self.betas)))
- # Prepare data
- X = build_design_matrix(X=stim, n_lag=self.dims[0], shift=self.shift, dtype=np.float32)
+ # Prepare data: build a design matrix for every channel.
+ Xs = self._build_design_matrices(stim)
y = trace.copy().astype(np.float32)
+ # Number of samples is the same across channels.
+ n_samples = next(iter(Xs.values())).shape[0]
+
# Folds
if self.kfold == 0:
- splits = [(X, y, None, None)]
+ splits = [(Xs, y, None, None)]
elif self.kfold == 1:
split_idx = int(y.size * 0.8)
- splits = [(X[:split_idx], y[:split_idx], X[split_idx:], y[split_idx:])]
+ splits = [(self._slice_design_matrices(Xs, slice(None, split_idx)),
+ y[:split_idx],
+ self._slice_design_matrices(Xs, slice(split_idx, None)),
+ y[split_idx:])]
else:
splits = []
- for train_idx, dev_idx in KFold(n_splits=self.kfold, shuffle=False).split(X, y):
+ for train_idx, dev_idx in KFold(n_splits=self.kfold, shuffle=False).split(np.arange(n_samples), y):
train_idx = train_idx[(train_idx < np.min(dev_idx)) | (train_idx > np.max(dev_idx) + self.burn_in)]
- splits += [(X[train_idx], y[train_idx], X[dev_idx], y[dev_idx])]
+ splits += [(self._slice_design_matrices(Xs, train_idx),
+ y[train_idx],
+ self._slice_design_matrices(Xs, dev_idx),
+ y[dev_idx])]
best_model = None
for idx_dfs, df in enumerate(dfs):
if self.verbose > 0:
print(f'############ Optimize for df={df} ############')
- for idx_kf, (X_train_k, y_train_k, X_dev_k, y_dev_k) in enumerate(splits):
+ for idx_kf, (Xs_train_k, y_train_k, Xs_dev_k, y_dev_k) in enumerate(splits):
if self.verbose > 0 and self.kfold > 1:
print(f"###### Fold: {idx_kf + 1}/{self.kfold} ######")
best_model, metrics_dev_opt_i = self.create_and_fit_glm(
- df=df, X_train=X_train_k, y_train=y_train_k, X_dev=X_dev_k, y_dev=y_dev_k,
+ df=df, Xs_train=Xs_train_k, y_train=y_train_k,
+ Xs_dev=Xs_dev_k, y_dev=y_dev_k,
alphas=self.alphas, betas=self.betas)
metrics_dev_opt[idx_kf, idx_dfs, :] = metrics_dev_opt_i
@@ -361,7 +543,7 @@ def compute_best_rf_model(self, trace: np.ndarray, stim: np.ndarray) -> tuple:
f'\tdf={best_df} and beta={best_beta:.4g}')
best_model, _ = self.create_and_fit_glm(
- df=best_df, X_train=X, y_train=y, X_dev=None, y_dev=None,
+ df=best_df, Xs_train=Xs, y_train=y, Xs_dev=None, y_dev=None,
alphas=self.alphas, betas=[best_beta])
best_model.metric_dev_opt = np.mean(metrics_dev_opt[:, best_df_idx, best_beta_idx])
@@ -398,86 +580,70 @@ def clip_dfs(dfs: np.ndarray, dims: int) -> np.ndarray:
warnings.warn(' All df_ts larger than dims, use only dims as df')
return np.array([dims])
- @staticmethod
- def _df_w2df_h(df_w: int, X_train: np.ndarray, hdims: int) -> int:
- """Convert width degrees of freedom to height degrees of freedom.
-
- Parameters
- ----------
- df_w : int
- Degrees of freedom for the width dimension.
- X_train : np.ndarray
- Training design matrix (used for shape reference).
- hdims : int
- Maximum allowed height degrees of freedom.
-
- Returns
- -------
- int
- Height degrees of freedom.
- """
- df_h = np.minimum(int(np.round(df_w * X_train.shape[1] / X_train.shape[2])), hdims)
- return df_h
-
- def create_and_fit_glm(self, df: tuple, X_train: np.ndarray, y_train: np.ndarray,
+ def create_and_fit_glm(self, df: tuple, Xs_train: dict, y_train: np.ndarray,
alphas, betas,
- X_dev: np.ndarray = None,
+ Xs_dev: dict = None,
y_dev: np.ndarray = None) -> tuple:
"""Create and fit a GLM model for given hyperparameters.
Parameters
----------
df : tuple
- Degrees of freedom per dimension (df_t, df_h, df_w) or (df_t,).
- X_train : np.ndarray
- Training design matrix.
+ Degrees of freedom per dimension.
+ Xs_train : dict
+ Per-channel training design matrices.
y_train : np.ndarray
Training response.
alphas : array-like
Candidate alpha values.
betas : array-like
Candidate beta values.
- X_dev : np.ndarray, optional
- Development design matrix. Default is None.
+ Xs_dev : dict, optional
+ Per-channel dev design matrices. Default is None.
y_dev : np.ndarray, optional
Development response. Default is None.
Returns
-------
tuple
- model : object
- Fitted GLM model.
- metric_dev_opt_hp_sets : list or np.ndarray
- Development set metrics for each HP combination.
+ model, metric_dev_opt_hp_sets
"""
- assert X_train.shape[0] == y_train.shape[0]
- if X_dev is not None:
+ # Sanity checks across channels.
+ first_X = next(iter(Xs_train.values()))
+ assert first_X.shape[0] == y_train.shape[0]
+ for name, X in Xs_train.items():
+ assert X.shape[0] == y_train.shape[0], f"channel {name} train length mismatch"
+
+ if Xs_dev is not None:
assert y_dev is not None
- assert X_dev.shape[0] == y_dev.shape[0]
- assert X_train.shape[1:] == X_dev.shape[1:]
+ first_Xd = next(iter(Xs_dev.values()))
+ assert first_Xd.shape[0] == y_dev.shape[0]
+ for name in Xs_train:
+ assert name in Xs_dev, f"channel {name} missing in dev set"
+ assert Xs_train[name].shape[1:] == Xs_dev[name].shape[1:]
model = self.create_glm(
- df=df, X_train=X_train, y_train=y_train, X_dev=X_dev)
+ df=df, Xs_train=Xs_train, y_train=y_train, Xs_dev=Xs_dev)
model, metric_dev_opt_hp_sets = self.fit_glm(
model=model, y_train=y_train, y_dev=y_dev, alphas=alphas, betas=betas)
return model, metric_dev_opt_hp_sets
- def create_glm(self, df: tuple, X_train: np.ndarray,
- y_train: np.ndarray, X_dev: np.ndarray) -> object:
+ def create_glm(self, df: tuple, Xs_train: dict,
+ y_train: np.ndarray, Xs_dev: dict) -> object:
"""Create and initialize a GLM model.
Parameters
----------
df : tuple
Degrees of freedom per dimension.
- X_train : np.ndarray
- Training design matrix.
+ Xs_train : dict
+ Per-channel training design matrices.
y_train : np.ndarray
Training response.
- X_dev : np.ndarray
- Development design matrix (or None).
+ Xs_dev : dict
+ Per-channel dev design matrices (or None).
Returns
-------
@@ -501,7 +667,7 @@ def create_glm(self, df: tuple, X_train: np.ndarray,
df[i] = dim
df = tuple(df)
- model = self.initialize_model(df=df, X_train=X_train, y_train=y_train, X_dev=X_dev)
+ model = self.initialize_model(df=df, Xs_train=Xs_train, y_train=y_train, Xs_dev=Xs_dev)
return model
def fit_glm(self, model: object, alphas, betas,
@@ -524,10 +690,7 @@ def fit_glm(self, model: object, alphas, betas,
Returns
-------
tuple
- model : object
- Fitted GLM model.
- metric_dev_opt_hp_sets : list or np.ndarray
- Development set metrics for each HP combination.
+ model, metric_dev_opt_hp_sets
"""
min_iters_other = self.min_iters // 5
@@ -562,20 +725,24 @@ def fit_glm(self, model: object, alphas, betas,
return model, metric_dev_opt_hp_sets
- def initialize_model(self, df: tuple, X_train: np.ndarray,
- y_train: np.ndarray, X_dev: np.ndarray = None) -> object:
+ def initialize_model(self, df: tuple, Xs_train: dict,
+ y_train: np.ndarray, Xs_dev: dict = None) -> object:
"""Initialize model parameters for faster optimization.
+ For multi-channel data, adds one design matrix per channel under
+ the channel's name, so the GLM forward pass becomes
+ y = nonlin( sum_c X_c @ w_c + intercept ).
+
Parameters
----------
df : tuple
- Degrees of freedom per dimension.
- X_train : np.ndarray
- Training design matrix.
+ Degrees of freedom per dimension. Shared across channels.
+ Xs_train : dict
+ Per-channel training design matrices.
y_train : np.ndarray
Training response.
- X_dev : np.ndarray, optional
- Development design matrix. Default is None.
+ Xs_dev : dict, optional
+ Per-channel dev design matrices. Default is None.
Returns
-------
@@ -585,12 +752,16 @@ def initialize_model(self, df: tuple, X_train: np.ndarray,
model = rfest.GLM(distr=self.distr, output_nonlinearity=self.output_nonlinearity)
model.burn_in = self.burn_in
- model.add_design_matrix(X_train, is_design_matrix=True, df=df, dims=self.dims, shift=self.shift,
- smooth='cr', filter_nonlinearity='none', name='stimulus')
- if X_dev is not None:
- model.add_design_matrix(X_dev, is_design_matrix=True, dims=self.dims, shift=self.shift,
- name='stimulus', kind='dev')
+ for name in self.channel_names:
+ model.add_design_matrix(
+ Xs_train[name], is_design_matrix=True, df=df, dims=self.dims, shift=self.shift,
+ smooth='cr', filter_nonlinearity='none', name=name)
+
+ if Xs_dev is not None:
+ model.add_design_matrix(
+ Xs_dev[name], is_design_matrix=True, dims=self.dims, shift=self.shift,
+ name=name, kind='dev')
model.initialize(num_subunits=self.num_subunits, dt=self.dt,
method=self.init_method,
@@ -620,18 +791,7 @@ def get_b_from_w(S: np.ndarray, w: np.ndarray) -> np.ndarray:
@staticmethod
def _get_model_info(model: object) -> str:
- """Format a summary string of key model hyperparameters and metrics.
-
- Parameters
- ----------
- model : object
- Fitted GLM model with attributes metric, metric_dev_opt, alpha, beta, df, dims.
-
- Returns
- -------
- str
- Human-readable info string.
- """
+ """Format a summary string of key model hyperparameters and metrics."""
info = f"{model.metric}={model.metric_dev_opt:.4f} "
info += f"for HP-set: alpha={model.alpha:.5g}, beta={model.beta:.5g}, df={model.df}, dims={model.dims}"
return info
@@ -670,26 +830,50 @@ def plot_rf_summary(rf: np.ndarray, quality_dict: dict, model_dict: dict,
title: str = "") -> tuple:
"""Plot tRF and sRFs and quality test.
+ For multi-channel RFs, this function recurses and produces one summary
+ figure per channel.
+
Parameters
----------
rf : np.ndarray
- Receptive field array, shape (n_t,) for 1D or (n_t, n_y, n_x) for 3D.
+ Receptive field array. Single-channel: (n_t,) or (n_t, n_y, n_x).
+ Multi-channel: (n_colors, n_t) or (n_colors, n_t, n_y, n_x).
quality_dict : dict
- Quality metrics dict, may contain 'score_permX', 'quality', 'corrcoef_*', etc.
+ Quality metrics dict.
model_dict : dict
- Model metadata dict, may contain 'metric_train', 'y_train', 'y_test', etc.
+ Model metadata dict, may contain 'channel_names' / 'n_colors' /
+ 'ws_per_channel' for multi-channel data.
title : str, optional
Figure title. Default is "".
Returns
-------
- tuple
- fig : matplotlib.figure.Figure
- The figure object.
- axs : np.ndarray
- Array of axes objects.
+ tuple or list of tuples
+ Single-channel: (fig, axs).
+ Multi-channel: list of (fig, axs), one per channel.
"""
-
+ n_colors = (model_dict or {}).get('n_colors', 1)
+
+ # Multi-channel: recurse, one figure per channel.
+ if n_colors > 1:
+ channel_names = model_dict.get('channel_names', [f'ch{i}' for i in range(n_colors)])
+ results = []
+ for c, name in enumerate(channel_names):
+ rf_c = rf[c]
+ md_c = dict(model_dict) # shallow copy
+ md_c['n_colors'] = 1 # prevent re-recursion
+ md_c['channel_names'] = [name]
+ ch_title = f"{title} [{name}]" if title else f"[{name}]"
+ res = _plot_rf_summary_single(rf_c, quality_dict, md_c, title=ch_title)
+ results.append(res)
+ return results
+
+ return _plot_rf_summary_single(rf, quality_dict, model_dict, title=title)
+
+
+def _plot_rf_summary_single(rf: np.ndarray, quality_dict: dict, model_dict: dict,
+ title: str = "") -> tuple:
+ """Single-channel RF summary plot. (Original plot_rf_summary logic.)"""
plot_training = model_dict is not None and 'metric_train' in model_dict
plot_y_train = model_dict is not None and 'y_train' in model_dict
plot_y_dev = model_dict is not None and 'y_dev' in model_dict
@@ -752,7 +936,7 @@ def plot_rf_summary(rf: np.ndarray, quality_dict: dict, model_dict: dict,
if not plot_training:
plt.tight_layout(rect=(0.01, 0.01, 0.99, 0.90), h_pad=0.2, w_pad=0.2)
- return
+ return fig, axs
if model_dict is not None and 'metric_train' in model_dict:
ax = axs[4]
diff --git a/djimaging/utils/receptive_fields/plot_rf_utils.py b/djimaging/utils/receptive_fields/plot_rf_utils.py
index a90e3bd..23e0192 100644
--- a/djimaging/utils/receptive_fields/plot_rf_utils.py
+++ b/djimaging/utils/receptive_fields/plot_rf_utils.py
@@ -91,7 +91,8 @@ def plot_rf_video(rf: np.ndarray, rf_time: np.ndarray, fps: int = 10):
Parameters
----------
rf : np.ndarray
- Receptive field array, shape (n_t, n_y, n_x) or (n_t, n_y, n_x, n_colors).
+ Receptive field array, shape (n_t, n_y, n_x), (n_t, n_y, n_x, n_colors),
+ or (n_t, n_features) for 1-D / color-only RFs.
rf_time : np.ndarray
Time axis, shape (n_t,).
fps : int, optional
@@ -109,32 +110,38 @@ def plot_rf_video(rf: np.ndarray, rf_time: np.ndarray, fps: int = 10):
if rf.ndim == 4:
rf = flatten_color_channels(rf)
- fig, ax = plt.subplots(1, 1, figsize=(5, 5))
- ax.set_aspect('equal')
-
- vabsmax = np.nanmax(np.abs(rf))
- im = ax.imshow(rf[0], cmap='coolwarm', vmin=-vabsmax, vmax=vabsmax, origin='lower')
-
- def update(frame: int):
- """Update the animation frame.
-
- Parameters
- ----------
- frame : int
- Current frame index.
+ if rf.ndim == 2:
+ # 1-D or color-only RF: animate as a line plot
+ fig, ax = plt.subplots(1, 1, figsize=(6, 3))
+ vabsmax = np.nanmax(np.abs(rf))
+ line, = ax.plot(rf[0])
+ ax.set_ylim(-vabsmax * 1.1, vabsmax * 1.1)
+ ax.set_title(f"t={rf_time[0]:.3f}")
+
+ def update(frame: int):
+ if frame < len(rf):
+ line.set_ydata(rf[frame])
+ ax.set_title(f"t={rf_time[frame]:.3f}")
+ else:
+ line.set_ydata(np.zeros_like(rf[0]))
+ ax.set_title("pause")
+ return line,
+ else:
+ # 2-D spatial RF: animate as imshow
+ fig, ax = plt.subplots(1, 1, figsize=(5, 5))
+ ax.set_aspect('equal')
- Returns
- -------
- tuple
- Updated image artist.
- """
- if frame < len(rf):
- im.set_data(rf[frame])
- ax.set_title(f"t={rf_time[frame]:.3f}")
- else:
- im.set_data(np.zeros_like(rf[0]))
- ax.set_title("pause")
- return im,
+ vabsmax = np.nanmax(np.abs(rf))
+ im = ax.imshow(rf[0], cmap='coolwarm', vmin=-vabsmax, vmax=vabsmax, origin='lower')
+
+ def update(frame: int):
+ if frame < len(rf):
+ im.set_data(rf[frame])
+ ax.set_title(f"t={rf_time[frame]:.3f}")
+ else:
+ im.set_data(np.zeros_like(rf[0]))
+ ax.set_title("pause")
+ return im,
plt.close(fig)
diff --git a/djimaging/utils/receptive_fields/preprocess_rf_utils.py b/djimaging/utils/receptive_fields/preprocess_rf_utils.py
index 20cdfd4..cdd1c27 100644
--- a/djimaging/utils/receptive_fields/preprocess_rf_utils.py
+++ b/djimaging/utils/receptive_fields/preprocess_rf_utils.py
@@ -11,7 +11,7 @@
def prepare_noise_data(stim: np.ndarray, triggertimes: np.ndarray,
trace: np.ndarray, tracetime: np.ndarray,
- ntrigger_per_frame: int = 1,
+ nframes_per_trigger: int = 1,
fupsample_trace: int = None,
fupsample_stim: int = None,
fit_kind: str = 'trace',
@@ -35,7 +35,7 @@ def prepare_noise_data(stim: np.ndarray, triggertimes: np.ndarray,
Neural response trace, shape (n_timepoints,).
tracetime : np.ndarray
Timestamps for the trace, shape (n_timepoints,).
- ntrigger_per_frame : int, optional
+ nframes_per_trigger : int, optional
Number of triggers per stimulus frame. Default is 1.
fupsample_trace : int, optional
Upsampling factor for the trace. Default is None.
@@ -80,7 +80,7 @@ def prepare_noise_data(stim: np.ndarray, triggertimes: np.ndarray,
# Resampling and filtering
stimtime, stim = preprocess_stimulus(
- stim, triggertimes, ntrigger_per_frame, fupsample_stim)
+ stim, triggertimes, nframes_per_trigger, fupsample_stim)
tracetime, trace = preprocess_trace(
tracetime, trace, fupsample_trace, fit_kind, lowpass_cutoff, pre_blur_sigma_s, dt_rtol)
@@ -102,7 +102,7 @@ def prepare_noise_data(stim: np.ndarray, triggertimes: np.ndarray,
def preprocess_stimulus(stim: np.ndarray, triggertimes: np.ndarray,
- ntrigger_per_frame: int,
+ nframes_per_trigger: int = 1,
fupsample_stim: int = None) -> tuple[np.ndarray, np.ndarray]:
"""Preprocess the stimulus by aligning to trigger times and optionally upsampling.
@@ -112,8 +112,9 @@ def preprocess_stimulus(stim: np.ndarray, triggertimes: np.ndarray,
Stimulus array, shape (n_frames, ...).
triggertimes : np.ndarray
Trigger timestamps in seconds, shape (n_triggers,).
- ntrigger_per_frame : int
- Number of triggers per stimulus frame.
+ nframes_per_trigger : int, optional
+ Number of stimulus frames per trigger. Use when triggers are less
+ frequent than frames (e.g. one trigger every N frames). Default is 1.
fupsample_stim : int, optional
Upsampling factor for the stimulus. Default is None.
@@ -130,17 +131,17 @@ def preprocess_stimulus(stim: np.ndarray, triggertimes: np.ndarray,
ValueError
If there are more triggertimes than stimulus frames.
"""
- if ntrigger_per_frame > 1:
- stimtime = np.repeat(triggertimes, ntrigger_per_frame)
+ if nframes_per_trigger > 1:
+ stimtime = np.repeat(triggertimes, nframes_per_trigger)
trigger_dt, _ = get_mean_dt(triggertimes, rtol_error=np.inf, rtol_warning=0.5)
- stimtime += np.tile(np.arange(ntrigger_per_frame) * trigger_dt / ntrigger_per_frame,
- stimtime.size // ntrigger_per_frame)
+ stimtime += np.tile(np.arange(nframes_per_trigger) * trigger_dt / nframes_per_trigger,
+ stimtime.size // nframes_per_trigger)
else:
stimtime = triggertimes
if stim.shape[0] < stimtime.size:
raise ValueError(
- f"More triggertimes than expected: stim-len {stim.shape[0]} != stimtime-len {stimtime.size}")
+ f"More triggertimes than expected: stim-len {stim.shape[0]} != stimtime-len {stimtime.size} for {nframes_per_trigger=} and {len(triggertimes)=}")
elif stim.shape[0] > stimtime.size:
warnings.warn(f"Less triggertimes than expected: stim-len {stim.shape[0]} != stimtime-len {stimtime.size}")
stim = stim[:stimtime.size].copy()
diff --git a/djimaging/utils/scanm/roi_utils.py b/djimaging/utils/scanm/roi_utils.py
index a6bbf69..b46f4c2 100644
--- a/djimaging/utils/scanm/roi_utils.py
+++ b/djimaging/utils/scanm/roi_utils.py
@@ -128,48 +128,6 @@ def get_roi2trace(
return roi2trace
-def compare_roi_masks(roi_mask1: np.ndarray, roi_mask2: np.ndarray, max_shift: int = 4) -> str:
- """Test if two ROI masks are the same, possibly up to a spatial shift.
-
- Parameters
- ----------
- roi_mask1 : np.ndarray
- First 2-D integer ROI mask in Igor format.
- roi_mask2 : np.ndarray
- Second 2-D integer ROI mask in Igor format.
- max_shift : int, optional
- Maximum number of pixels to try shifting in each direction when
- testing for a shifted match. Default is 4.
-
- Returns
- -------
- str
- ``'same'`` if the masks are identical, ``'shifted'`` if they match
- after a spatial shift within ``max_shift``, or ``'different'``
- otherwise.
- """
- assert_igor_format(roi_mask1)
- assert_igor_format(roi_mask2)
-
- if roi_mask1.shape != roi_mask2.shape:
- return 'different'
- if np.all(roi_mask1 == roi_mask2):
- return 'same'
-
- max_shift_x = np.minimum(max_shift, roi_mask1.shape[0] - 2)
- max_shift_y = np.minimum(max_shift, roi_mask1.shape[1] - 2)
-
- for dx in range(-max_shift_x, max_shift_x + 1):
- for dy in range(-max_shift_y, max_shift_y + 1):
- shifted1 = roi_mask1[dx:, dy:]
- dx = -roi_mask2.shape[0] if dx == 0 else dx # Handle zero case
- dy = -roi_mask2.shape[0] if dy == 0 else dy
- shifted2 = roi_mask2[:-dx, :-dy]
- if np.all(shifted1 == shifted2):
- return 'shifted'
- return 'different'
-
-
def get_roi_center(roi_mask: np.ndarray, roi_id: int) -> Tuple[float, float]:
"""Return the centre of mass of a single ROI in pixel coordinates.
diff --git a/pyproject.toml b/pyproject.toml
index b265cad..ff6bf83 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,34 +9,36 @@ description = "DataJoint for 2P imaging data"
authors = [{ name = "Jonathan Oesterle", email = "jonathan.oesterle@uni-tuebingen.de" }]
requires-python = ">=3.10"
dependencies = [
- "datajoint>=0.13",
- "numpy",
- "h5py",
- "scipy",
- "cached_property",
- "matplotlib",
- "pandas",
- "seaborn",
- "scikit-learn",
- "opencv-python",
- "astropy",
- "ipython",
- "numba",
- "scanm-support",
+ "datajoint>=2.3.1,<2.4",
+ # The converted classifier pickle references numpy._core.numeric (NumPy 2).
+ "numpy>=2.0.0",
+ # Compiled dependencies must support the NumPy 2 ABI.
+ "h5py>=3.11.0",
+ "scipy>=1.13.0",
+ "matplotlib>=3.8.4",
+ "pandas>=2.2.2",
+ "seaborn>=0.13.2",
+ # Saved classifiers additionally require their exact scikit-learn version.
+ "scikit-learn>=1.7.2",
+ "opencv-python>=4.10.0.84",
+ "astropy>=6.1.0",
+ "ipython>=8.12.3",
+ "numba>=0.60.0",
+ "scanm-support>=0.0.1",
]
[project.optional-dependencies]
autorois = [
- "pytorch_lightning",
- "torch",
- "ipywidgets",
- "ipycanvas",
+ "pytorch_lightning>=2.4.0",
+ "torch>=2.4.1",
+ "ipywidgets>=8.0.0",
+ "ipycanvas>=0.13.0",
]
rf = [
"rfest>=2.1.2",
]
dev = [
- "pytest",
+ "pytest>=9.0.3",
]
[tool.setuptools.packages.find]
@@ -44,6 +46,18 @@ where = ["."]
[tool.pytest.ini_options]
markers = ["dependency: mark a test as a dependency"]
+filterwarnings = [
+ "error:fetch\\(\\) is deprecated.*:DeprecationWarning",
+ "error:ERD is deprecated.*:DeprecationWarning",
+]
+
+[tool.uv]
+# Minimum patched versions for transitive dependencies with security advisories.
+constraint-dependencies = [
+ "pillow>=12.3.0",
+ "aiohttp>=3.14.3",
+ "idna>=3.15",
+]
[tool.uv.sources]
scanm-support = { git = "https://github.com/eulerlab/ScanM_support.git" }
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 33da3dc..ee3e713 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -1,12 +1,34 @@
import os
import uuid
+from pathlib import Path
import datajoint as dj
import pytest
@pytest.fixture(scope="session")
-def tutorial_schema():
+def dj_test_stores(tmp_path_factory: pytest.TempPathFactory, tutorial_data_dir: Path) -> dict[str, Path]:
+ root = tmp_path_factory.mktemp("datajoint-v2-stores")
+ stores = {
+ "reference": tutorial_data_dir,
+ "processed": root / "processed",
+ }
+ for path in stores.values():
+ path.mkdir(exist_ok=True)
+
+ dj.config["stores"] = {
+ "default": "processed",
+ "filepath_default": "reference",
+ **{name: {"protocol": "file", "location": str(path)} for name, path in stores.items()},
+ }
+ download_path = root / "downloads"
+ download_path.mkdir()
+ dj.config["download_path"] = str(download_path)
+ return stores
+
+
+@pytest.fixture(scope="session")
+def tutorial_schema(dj_test_stores):
dj.config["database.host"] = os.environ.get("DJ_HOST", "127.0.0.1")
dj.config["database.port"] = int(os.environ.get("DJ_PORT", "3306"))
dj.config["database.user"] = os.environ.get("DJ_USER", "root")
@@ -18,6 +40,20 @@ def tutorial_schema():
from djimaging.schemas import tutorial_schema as schema_module
from djimaging.utils.dj_utils import activate_schema
+ @schema_module.schema
+ class CodecRoundTrip(dj.Manual):
+ definition = """
+ object_id : int32
+ ---
+ source_file :
+ attachment :
+ small_object :
+ large_object :
+ array :
+ """
+
+ schema_module.CodecRoundTrip = CodecRoundTrip
+
try:
activate_schema(schema_module.schema, schema_name=schema_name)
yield schema_module
diff --git a/tests/integration/test_core_schema.py b/tests/integration/test_core_schema.py
index da47980..de9d345 100644
--- a/tests/integration/test_core_schema.py
+++ b/tests/integration/test_core_schema.py
@@ -10,7 +10,18 @@
def test_all_tutorial_tables_initialize(tutorial_schema):
- assert tutorial_schema.schema.list_tables()
+ tables = tutorial_schema.schema.list_tables()
+ assert tables
+ for name in ("ClassifierV2", "Baden16TracesV2", "CelltypeAssignmentV2"):
+ assert getattr(tutorial_schema, name).table_name in tables
+
+
+def test_full_rgc_tables_initialize(tutorial_schema):
+ from djimaging.schemas import full_rgc_schema
+
+ tables = tutorial_schema.schema.list_tables()
+ for name in ('FastStaParams', 'FastSta', 'FastStaQuality', 'SplitRF'):
+ assert getattr(full_rgc_schema, name).table_name in tables
def test_user_info_round_trip(tutorial_schema, tmp_path):
diff --git a/tests/integration/test_storage_codecs.py b/tests/integration/test_storage_codecs.py
new file mode 100644
index 0000000..bdbd76c
--- /dev/null
+++ b/tests/integration/test_storage_codecs.py
@@ -0,0 +1,60 @@
+import os
+from pathlib import Path
+
+import datajoint as dj
+import numpy as np
+import pytest
+
+from djimaging.utils.dj_storage import load_array, local_path, open_object
+
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("DJ_TEST_MYSQL") != "1",
+ reason="requires the MySQL integration-test service",
+)
+
+
+def test_datajoint_v2_storage_round_trip_and_deletion(tutorial_schema, dj_test_stores, tmp_path: Path):
+ source = dj_test_stores["reference"] / "raw" / "source.bin"
+ source.parent.mkdir()
+ source.write_bytes(b"source payload")
+
+ attachment = tmp_path / "model.txt"
+ attachment.write_text("model payload")
+
+ array = np.arange(24, dtype=np.float32).reshape(4, 6)
+ large_object = {"names": ["a", "b"], "values": np.arange(10)}
+ table = tutorial_schema.CodecRoundTrip()
+ table.insert1(
+ {
+ "object_id": 1,
+ "source_file": "raw/source.bin",
+ "attachment": attachment,
+ "small_object": {"enabled": True},
+ "large_object": large_object,
+ "array": array,
+ }
+ )
+
+ row = (table & {"object_id": 1}).fetch1()
+ assert isinstance(row["source_file"], dj.ObjectRef)
+ assert isinstance(row["array"], dj.NpyRef)
+ assert row["small_object"] == {"enabled": True}
+ assert row["large_object"]["names"] == large_object["names"]
+ np.testing.assert_array_equal(row["large_object"]["values"], large_object["values"])
+ np.testing.assert_array_equal(load_array(row["array"]), array)
+
+ with open_object(row["source_file"]) as file:
+ assert file.read() == b"source payload"
+ with local_path(row["source_file"]) as path:
+ assert path == source
+ assert Path(row["attachment"]).read_text() == "model payload"
+
+ npy_path = row["array"].path
+ (table & {"object_id": 1}).delete(prompt=False)
+ assert len(table) == 0
+ assert source.exists(), "filepath data is user-managed and must not be deleted with a row"
+
+ processed_report = dj.gc.GarbageCollector(tutorial_schema.schema, store="processed").collect(dry_run=True)
+ assert npy_path in processed_report["orphaned_schema_paths"]
+ assert processed_report["hash_paths_orphaned"] >= 1
diff --git a/tests/integration/test_tutorial_pipeline.py b/tests/integration/test_tutorial_pipeline.py
index 0f33a3d..327838d 100644
--- a/tests/integration/test_tutorial_pipeline.py
+++ b/tests/integration/test_tutorial_pipeline.py
@@ -3,7 +3,9 @@
import h5py
import numpy as np
import pytest
+from sklearn.dummy import DummyClassifier
+from djimaging.tables.classifier_v2.rgc_classifier_v2 import save_classifier_to_file
from tests.fixtures.random_utils import numpy_seed
@@ -13,7 +15,7 @@
)
-def test_tutorial_pipeline(tutorial_schema, tutorial_data_dir):
+def test_tutorial_pipeline(tutorial_schema, tutorial_data_dir, dj_test_stores):
experiment_key = {"experimenter": "synthetic"}
tutorial_schema.UserInfo().upload_user(
@@ -34,7 +36,13 @@ def test_tutorial_pipeline(tutorial_schema, tutorial_data_dir):
tutorial_schema.RawDataParams().add_default(experimenter_list=["synthetic"])
tutorial_schema.Experiment().rescan_filesystem(restrictions=experiment_key, verboselvl=0)
+
+ processed = dj_test_stores["processed"]
+ external_files_before = {path for path in processed.rglob("*") if path.is_file()}
tutorial_schema.Field().rescan_filesystem(restrictions=experiment_key, verboselvl=0)
+ assert {path for path in processed.rglob("*") if path.is_file()} == external_files_before, (
+ "Field channel averages must stay in the database"
+ )
tutorial_schema.Stimulus().add_nostim(skip_duplicates=True)
tutorial_schema.Stimulus().add_chirp(
@@ -57,11 +65,13 @@ def test_tutorial_pipeline(tutorial_schema, tutorial_data_dir):
pix_n_y=15,
pix_scale_x_um=30,
pix_scale_y_um=30,
+ stim_path=tutorial_data_dir / "resources" / "noise.h5",
stim_trace=noise_stimulus,
skip_duplicates=True,
)
tutorial_schema.Stimulus().add_movingbar(skip_duplicates=True)
+ external_files_before = {path for path in processed.rglob("*") if path.is_file()}
tutorial_schema.Presentation().populate(display_progress=False)
tutorial_schema.RoiMask().rescan_filesystem(
restrictions=experiment_key,
@@ -69,6 +79,7 @@ def test_tutorial_pipeline(tutorial_schema, tutorial_data_dir):
roi_mask_dir="AutoROIs",
)
tutorial_schema.Roi().populate(experiment_key, display_progress=False)
+
tutorial_schema.Traces().populate(experiment_key, display_progress=False)
tutorial_schema.PreprocessParams().add_default(skip_duplicates=True)
tutorial_schema.PreprocessTraces().populate(experiment_key, display_progress=False)
@@ -79,6 +90,12 @@ def test_tutorial_pipeline(tutorial_schema, tutorial_data_dir):
with numpy_seed(42):
tutorial_schema.OsDsIndexes().populate(experiment_key, display_progress=False)
+ external_files_after = {path for path in processed.rglob("*") if path.is_file()}
+ assert external_files_after == external_files_before, (
+ "Presentation channel averages, ROI masks, traces, snippets, averages, "
+ "and response metrics must stay in the database"
+ )
+
tutorial_schema.OpticDisk().populate(experiment_key, display_progress=False)
tutorial_schema.RelativeFieldLocation().populate(experiment_key, display_progress=False)
tutorial_schema.RetinalFieldLocation().populate(experiment_key, display_progress=False)
@@ -110,7 +127,37 @@ def test_tutorial_pipeline(tutorial_schema, tutorial_data_dir):
}
assert not count_mismatches, f"Unexpected table counts: {count_mismatches}"
- assert np.all((tutorial_schema.Presentation() & experiment_key).fetch("trigger_valid") == 1)
- assert np.all(np.isfinite((tutorial_schema.ChirpQI() & experiment_key).fetch("qidx")))
- assert np.all(np.isfinite((tutorial_schema.OsDsIndexes() & experiment_key).fetch("ds_index")))
- assert np.all(np.isfinite((tutorial_schema.OsDsIndexes() & experiment_key).fetch("os_index")))
+ assert np.all((tutorial_schema.Presentation() & experiment_key).to_arrays("trigger_valid") == 1)
+ assert np.all(np.isfinite((tutorial_schema.ChirpQI() & experiment_key).to_arrays("qidx")))
+ assert np.all(np.isfinite((tutorial_schema.OsDsIndexes() & experiment_key).to_arrays("ds_index")))
+ assert np.all(np.isfinite((tutorial_schema.OsDsIndexes() & experiment_key).to_arrays("os_index")))
+
+ with numpy_seed(42):
+ tutorial_schema.Baden16TracesV2().populate(experiment_key, display_progress=False)
+ baden_traces = (tutorial_schema.Baden16TracesV2() & experiment_key).to_dicts()
+ assert len(baden_traces) == 2
+
+ # Exercise classifier loading and population with a model trained in this environment.
+ train_x = np.zeros((75, 4))
+ train_y = np.arange(1, 76)
+ classifier = DummyClassifier(strategy="uniform").fit(train_x, train_y)
+ classifier_file = tutorial_data_dir / "resources" / "test_classifier.pkl"
+ save_classifier_to_file(
+ classifier=classifier,
+ chirp_feats=np.ones((baden_traces[0]["preproc_chirp"].size, 1)),
+ bar_feats=np.ones((baden_traces[0]["preproc_bar"].size, 1)),
+ feature_names=["chirp", "bar", "ds", "size"],
+ train_x=train_x,
+ train_y=train_y,
+ y_names={label: str(label) for label in train_y},
+ classifier_file=classifier_file,
+ )
+ tutorial_schema.ClassifierV2().add(classifier_file=classifier_file)
+ summary = tutorial_schema.CelltypeAssignmentV2().populate(
+ experiment_key, {"classifier_id": 1}, display_progress=False,
+ )
+ assert summary == {"success_count": 1, "error_list": []}
+ assignments = (tutorial_schema.CelltypeAssignmentV2() & experiment_key).to_dicts()
+ assert len(assignments) == 2
+ for row in assignments:
+ np.testing.assert_allclose(row["probs_per_cluster"], np.full(75, 1 / 75))
diff --git a/tests/tables/test_datajoint_v2_definitions.py b/tests/tables/test_datajoint_v2_definitions.py
new file mode 100644
index 0000000..898e6ba
--- /dev/null
+++ b/tests/tables/test_datajoint_v2_definitions.py
@@ -0,0 +1,73 @@
+import importlib
+import inspect
+import pkgutil
+import re
+
+import datajoint as dj
+from datajoint.declare import NATIVE_TYPES, attribute_parser, is_foreign_key, match_type
+
+import djimaging.tables
+from djimaging.tables import classifier_v2, core, location, misc
+
+
+def _table_classes():
+ classes = set()
+ for module_info in pkgutil.walk_packages(djimaging.tables.__path__, djimaging.tables.__name__ + "."):
+ module = importlib.import_module(module_info.name)
+ for _, table_class in inspect.getmembers(module, inspect.isclass):
+ if table_class.__module__ == module.__name__ and issubclass(table_class, dj.Table):
+ classes.add(table_class)
+ classes.update(
+ value
+ for value in table_class.__dict__.values()
+ if inspect.isclass(value) and issubclass(value, dj.Table)
+ )
+ return classes
+
+
+def _definition(table_class):
+ if table_class.__name__ in {"PresentationTemplate", "HighResTemplate"}:
+ field_flags = type(
+ "FieldFlags",
+ (),
+ {"incl_region": True, "incl_cond1": True, "incl_cond2": False, "incl_cond3": False},
+ )
+ table_class = type(f"Test{table_class.__name__}", (table_class,), {"field_table": field_flags})
+ return table_class().definition
+
+
+def test_every_table_definition_uses_datajoint_v2_types():
+ definitions = {table_class.__name__: _definition(table_class) for table_class in _table_classes()}
+ assert len(definitions) >= 75
+
+ for table_name, definition in definitions.items():
+ for line in re.split(r"\s*\n\s*", definition.strip()):
+ if (
+ not line
+ or line.startswith(("#", "---"))
+ or is_foreign_key(line)
+ or re.match(r"^(unique\s+)?index\s*\(", line, re.I)
+ ):
+ continue
+
+ parsed = attribute_parser.parse_string(line + "#", parse_all=True)
+ attribute_type = parsed.type.strip()
+ category = match_type(attribute_type)
+ assert category not in NATIVE_TYPES or attribute_type.lower() == "time", (
+ table_name,
+ parsed.name,
+ attribute_type,
+ )
+
+
+def test_server_file_fields_use_named_filepath_stores():
+ assert "field_data_file: " in core.FieldTemplate().definition
+ assert "pres_data_file :" in _definition(core.PresentationTemplate)
+ assert "highres_file :" in _definition(misc.HighResTemplate)
+ assert "od_fromfile :" in location.OpticDiskTemplate().definition
+ assert (
+ "field_data_file: "
+ in location.OutlineAbsTemplate.OutlineAbsField().definition
+ )
+ assert "stim_path=NULL :" in core.StimulusTemplate().definition
+ assert "classifier_file : " in classifier_v2.ClassifierV2Template().definition
diff --git a/tests/tables/test_datajoint_v2_runtime.py b/tests/tables/test_datajoint_v2_runtime.py
new file mode 100644
index 0000000..1024a69
--- /dev/null
+++ b/tests/tables/test_datajoint_v2_runtime.py
@@ -0,0 +1,217 @@
+import datetime
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import datajoint as dj
+import numpy as np
+import pytest
+from sklearn.dummy import DummyClassifier
+
+from djimaging.tables.classifier_v2 import celltype_assignment_v2
+from djimaging.tables.core.stim_logs import PresentationLogTemplate
+from djimaging.tables.motion_correction.motion_detection import MotionDetectionTemplate
+from djimaging.tables.receptivefield.fast_sta import FastStaTemplate
+from djimaging.tables.receptivefield.glm import _get_shared_model_shift
+
+
+class _Relation:
+ def __init__(self, rows, primary_key=()):
+ self.rows = list(rows)
+ self.primary_key = tuple(primary_key)
+
+ def __call__(self):
+ return self
+
+ def __len__(self):
+ return len(self.rows)
+
+ def __and__(self, restriction):
+ if isinstance(restriction, dj.AndList):
+ relation = self
+ for condition in restriction:
+ relation = relation & condition
+ return relation
+ elif isinstance(restriction, dict):
+ rows = [
+ row for row in self.rows
+ if all(row.get(name) == value for name, value in restriction.items() if name in row)
+ ]
+ elif isinstance(restriction, _Relation):
+ # Match related rows on all shared attributes (DataJoint's semijoin).
+ rows = [
+ row for row in self.rows
+ if any(
+ all(row[name] == other[name] for name in row.keys() & other.keys())
+ for other in restriction.rows
+ )
+ ]
+ elif restriction == 'aborted = 0':
+ rows = [row for row in self.rows if not row['aborted']]
+ else:
+ raise NotImplementedError(restriction)
+ return _Relation(rows, primary_key=self.primary_key)
+
+ def fetch1(self, *attrs):
+ if len(self.rows) != 1:
+ raise RuntimeError(f"Expected one row, found {len(self.rows)}")
+ row = self.rows[0]
+ if attrs == ('KEY',):
+ return {name: row[name] for name in self.primary_key}
+ values = tuple(row[name] for name in attrs)
+ return values[0] if len(values) == 1 else values
+
+ def to_dicts(self, order_by=None):
+ rows = [row.copy() for row in self.rows]
+ if order_by:
+ rows.sort(key=lambda row: tuple(row[name] for name in order_by))
+ return rows
+
+
+@pytest.mark.parametrize('table_class', [
+ celltype_assignment_v2.CelltypeAssignmentV2Template,
+ MotionDetectionTemplate,
+ FastStaTemplate,
+])
+@pytest.mark.parametrize('reserve_jobs', [False, True])
+def test_custom_populate_calls_datajoint_v2(table_class, reserve_jobs, monkeypatch):
+ train_x = np.zeros((75, 4))
+ train_y = np.arange(1, 76)
+ classifier = DummyClassifier().fit(train_x, train_y)
+ classifier_dict = {
+ 'classifier': classifier,
+ 'chirp_feats': np.ones((3, 1)),
+ 'bar_feats': np.ones((2, 1)),
+ 'feature_names': ['chirp', 'bar', 'ds', 'size'],
+ 'train_x': train_x,
+ 'train_y': train_y,
+ 'y_names': {label: str(label) for label in train_y},
+ }
+ monkeypatch.setattr(celltype_assignment_v2, 'load_classifier_from_file',
+ Mock(return_value=classifier_dict))
+ rows = [{'model_id': 1, 'variant': 1, 'classifier_file': 'classifier.pkl'}]
+ restrictions = ()
+ if reserve_jobs:
+ # Both conditions must hold when selecting the model/parameter set.
+ rows += [dict(rows[0], model_id=2), dict(rows[0], variant=2)]
+ restrictions = ({'model_id': 1}, {'variant': 1})
+
+ params_table = _Relation([
+ dict(row, x_stimulus=np.ones((4, 2)), dt=0.1, rf_time=np.arange(2),
+ burn_in=1, shift=0, fit_kind='trace', dims=(2,),
+ fupsample_stim=1, fupsample_trace=1, lowpass_cutoff=0,
+ pre_blur_sigma_s=0, post_blur_sigma_s=0)
+ for row in rows
+ ])
+ params_table.stimulus_table = _Relation([
+ dict(row, stim_dict={'nframes_per_trigger': 2}) for row in rows
+ ])
+ summary = {'success_count': 1, 'error_list': []}
+ worker = Mock(return_value=summary)
+ table = type('TestPopulate', (table_class,), {
+ 'connection': SimpleNamespace(in_transaction=False),
+ 'classifier_table': _Relation(rows),
+ 'params_table': params_table,
+ '_populate_direct': worker,
+ '_populate_distributed': worker,
+ })()
+ make_kwargs = {}
+ # Execute the real DataJoint populate() entry point; only its workers are stubbed.
+ result = table.populate(
+ *restrictions, reserve_jobs=reserve_jobs, make_kwargs=make_kwargs,
+ max_calls=2, display_progress=True, priority=3, refresh=False,
+ )
+
+ assert result is summary
+ worker.assert_called_once()
+ assert worker.call_args.args == restrictions
+ forwarded = worker.call_args.kwargs
+ assert forwarded['max_calls'] == 2
+ assert forwarded['display_progress'] is True
+ assert forwarded['processes'] == 1
+ if reserve_jobs:
+ assert forwarded['priority'] == 3
+ assert forwarded['refresh'] is False
+ if table_class is celltype_assignment_v2.CelltypeAssignmentV2Template:
+ assert forwarded['make_kwargs']['classifier'] is classifier
+ assert forwarded['make_kwargs']['chirp_feats'] is classifier_dict['chirp_feats']
+ assert forwarded['make_kwargs']['bar_feats'] is classifier_dict['bar_feats']
+ elif table_class is FastStaTemplate:
+ sta_params = forwarded['make_kwargs']['sta_params']
+ assert sta_params['dt'] == 0.1
+ assert sta_params['nframes_per_trigger'] == 2
+ np.testing.assert_array_equal(sta_params['rf_time'], np.arange(2))
+ np.testing.assert_array_equal(sta_params['x_stimulus'], np.ones((4, 2)))
+ assert sta_params['x_stimulus'].dtype == np.float32
+ assert make_kwargs == {}
+
+
+def test_get_shared_model_shift_supports_multiple_channels():
+ assert _get_shared_model_shift({
+ 'shift': {'ch0': -4, 'ch1': -4},
+ 'channel_names': ['ch0', 'ch1'],
+ }) == -4
+
+
+def test_get_shared_model_shift_rejects_different_channel_shifts():
+ with pytest.raises(ValueError, match="different shifts"):
+ _get_shared_model_shift({
+ 'shift': {'ch0': -4, 'ch1': -3},
+ 'channel_names': ['ch0', 'ch1'],
+ })
+
+
+def test_presentation_log_uses_configured_scan_params_table():
+ inserted = []
+ key = {'experiment_id': 1, 'stim_name': 'noise', 'presentation_id': 2}
+ log_rows = [
+ {
+ 'experiment_id': 1,
+ 'log_idx': 0,
+ 'stim_idx': 0,
+ 'stim_md5': 'abc',
+ 'stim_file_name': 'noise.py',
+ 'aborted': False,
+ 't_start': datetime.time(10, 0),
+ 't_end': datetime.time(10, 1),
+ },
+ {
+ 'experiment_id': 1,
+ 'log_idx': 0,
+ 'stim_idx': 1,
+ 'stim_md5': 'abc',
+ 'stim_file_name': 'noise.py',
+ 'aborted': False,
+ 't_start': datetime.time(11, 0),
+ 't_end': datetime.time(11, 1),
+ },
+ ]
+ table = SimpleNamespace(
+ exp_table=_Relation([{'experiment_id': 1}], primary_key=('experiment_id',)),
+ stimulus_table=_Relation([{
+ 'stim_name': 'noise',
+ 'stim_hash': 'abc',
+ 'stim_dict': {},
+ }]),
+ log_table=_Relation(log_rows),
+ scan_params_table=_Relation([{
+ **key,
+ 'scan_params_dict': {
+ 'datestamp_y_m_d': '2026-08-05',
+ 'timestamp_h_m_s_ms': '11-00-05-000',
+ },
+ }]),
+ presentation_table=object(),
+ insert1=inserted.append,
+ )
+
+ PresentationLogTemplate.make(table, key)
+
+ assert inserted == [{
+ **key,
+ 'log_idx': 0,
+ 'stim_idx': 1,
+ 'match_method': 'md5+time',
+ 't_start': datetime.time(11, 0),
+ 't_end': datetime.time(11, 1),
+ 'aborted': False,
+ }]
diff --git a/tests/tables/test_definitions_other_tables.py b/tests/tables/test_definitions_other_tables.py
index 8c87e13..1b28371 100644
--- a/tests/tables/test_definitions_other_tables.py
+++ b/tests/tables/test_definitions_other_tables.py
@@ -1,5 +1,4 @@
-import djimaging.tables.classifier.celltype_assignment
-from djimaging.tables import classifier, response, location, misc
+from djimaging.tables import classifier_v2, response, location, receptivefield
from djimaging.utils.dj_utils import get_input_tables, get_class_attributes
@@ -8,6 +7,7 @@ def _test_definition(djclass):
class_attrs = get_class_attributes(djclass)
for table in tables:
+ table = table.split('.', 1)[0].removesuffix('()')
assert table == 'master' or table in class_attrs, table
for class_attr in class_attrs:
@@ -30,26 +30,30 @@ def test_definition_ChirpQITemplate():
# orientation
-def test_definition_OsDsIndexesTemplateV1():
- _test_definition(response.OsDsIndexesTemplateV1)
+def test_definition_OsDsIndexesTemplate():
+ _test_definition(response.OsDsIndexesTemplate)
-def test_definition_OsDsIndexesTemplateV2():
- _test_definition(response.OsDsIndexesTemplateV2)
+def test_definition_ChirpSurroundTemplateV2():
+ _test_definition(response.ChirpSurroundTemplateV2)
-# rgc_classifier
-def test_definition_ClassifierTemplate():
- _test_definition(classifier.ClassifierTemplate)
+# pretrained classifier
+def test_definition_ClassifierV2Template():
+ _test_definition(classifier_v2.ClassifierV2Template)
-def test_definition_ClassifierTrainingDataTemplate():
- _test_definition(classifier.ClassifierTrainingDataTemplate)
+def test_definition_Baden16TracesV2Template():
+ _test_definition(classifier_v2.Baden16TracesV2Template)
-def test_definition_CelltypeAssignmentTemplate():
- _test_definition(classifier.CelltypeAssignmentTemplate)
+def test_definition_CelltypeAssignmentV2Template():
+ _test_definition(classifier_v2.CelltypeAssignmentV2Template)
-def test_definition_ClassifierMethodTemplate():
- _test_definition(classifier.ClassifierMethodTemplate)
+def test_definition_FastStaParamsTemplate():
+ _test_definition(receptivefield.FastStaParamsTemplate)
+
+
+def test_definition_FastStaTemplate():
+ _test_definition(receptivefield.FastStaTemplate)
diff --git a/tests/tables/test_orientation.py b/tests/tables/test_orientation.py
index 6305f82..4be7675 100644
--- a/tests/tables/test_orientation.py
+++ b/tests/tables/test_orientation.py
@@ -5,7 +5,7 @@
import numpy as np
import pytest
-from djimaging.tables.response.movingbar.orientation_utils_v2 import compute_os_ds_idxs
+from djimaging.tables.response.movingbar.orientation_utils import compute_os_ds_idxs
_TEST_DATA_PATH = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', 'test_data'))
diff --git a/tests/tables/test_orientation_synthetic.py b/tests/tables/test_orientation_synthetic.py
index 0e7d4fc..5f54b61 100644
--- a/tests/tables/test_orientation_synthetic.py
+++ b/tests/tables/test_orientation_synthetic.py
@@ -1,6 +1,6 @@
import numpy as np
-from djimaging.tables.response.movingbar.orientation_utils_v2 import compute_os_ds_idxs
+from djimaging.tables.response.movingbar.orientation_utils import compute_os_ds_idxs
from tests.fixtures.random_utils import numpy_seed
diff --git a/tests/tables/test_sta.py b/tests/tables/test_sta.py
index 0fa4e9f..388a99b 100644
--- a/tests/tables/test_sta.py
+++ b/tests/tables/test_sta.py
@@ -105,70 +105,6 @@ def test_sta_fit():
assert fit_mse < np.mean(random_mses) - 2 * np.std(random_mses)
-@pytest.mark.skipif(rfest is None, reason="requires rfest")
-def test_sta_fit_single_batch():
- np.random.seed(13488)
-
- w_true, X, y, dt, dims = rfest.simulate.generate_data_3d_stim(
- stim_noise='white', rf_kind='gauss', response_noise='none', design_matrix=False,
- n_stim_frames=500, n_reps_per_frame=3, shift=0)
-
- assert w_true.shape[1:] == X.shape[1:]
-
- w_fit, _ = fit_rf_utils._compute_linear_rf_single_batch(
- x_train=X, y_train=y, kind='sta', dim_t=w_true.shape[0], shift=0, burn_in=w_true.shape[0] - 1,
- threshold_pred=False, is_spikes=False, dtype=y.dtype)
-
- w_fit = w_fit.reshape(w_true.shape)
-
- fit_mse, random_mses = compare_fits(w_true=w_true, w_fit=w_fit)
- assert fit_mse < np.mean(random_mses) - 2 * np.std(random_mses)
-
-
-@pytest.mark.skipif(rfest is None, reason="requires rfest")
-def test_sta_fit_batchwise():
- np.random.seed(13488)
-
- w_true, X, y, dt, dims = rfest.simulate.generate_data_3d_stim(
- stim_noise='white', rf_kind='gauss', response_noise='none', design_matrix=False,
- n_stim_frames=500, n_reps_per_frame=3, shift=0)
-
- assert w_true.shape[1:] == X.shape[1:]
-
- w_fit, _ = fit_rf_utils._compute_sta_batchwise(
- x_train=X, y_train=y, kind='sta', dim_t=w_true.shape[0], shift=0, burn_in=w_true.shape[0] - 1,
- threshold_pred=False, is_spikes=False, dtype=y.dtype, batch_size=3)
-
- fit_mse, random_mses = compare_fits(w_true=w_true, w_fit=w_fit)
- assert fit_mse < np.mean(random_mses) - 2 * np.std(random_mses)
-
-
-@pytest.mark.skipif(rfest is None, reason="requires rfest")
-def test_sta_fit_single_batch_vs_batchwise():
- np.random.seed(13488)
-
- w_true, X, y, dt, dims = rfest.simulate.generate_data_3d_stim(
- stim_noise='white', rf_kind='gauss', response_noise='none', design_matrix=False,
- n_stim_frames=500, n_reps_per_frame=3, shift=0)
-
- assert w_true.shape[1:] == X.shape[1:]
-
- w_fit, y_pred = fit_rf_utils._compute_linear_rf_single_batch(
- x_train=X, y_train=y, kind='sta', dim_t=w_true.shape[0], shift=0, burn_in=w_true.shape[0] - 1,
- threshold_pred=False, is_spikes=False, dtype=y.dtype)
-
- w_fit = w_fit.reshape(w_true.shape)
-
- for batch_size in [1, 7, 400, 1000]:
- w_fit2, y_pred2 = fit_rf_utils._compute_sta_batchwise(
- x_train=X, y_train=y, kind='sta', dim_t=w_true.shape[0], shift=0, burn_in=w_true.shape[0] - 1,
- threshold_pred=False, is_spikes=False, dtype=y.dtype, batch_size=batch_size)
-
- w_fit2 = w_fit.reshape(w_true.shape)
- assert np.allclose(w_fit, w_fit2)
- assert np.allclose(y_pred, y_pred2)
-
-
@pytest.mark.skipif(rfest is None, reason="requires rfest")
def test_sta_fit_few_datapoints():
np.random.seed(364)
@@ -203,7 +139,7 @@ def test_sta_response():
stim_noise='white', rf_kind='gauss', response_noise='gaussian', design_matrix=True,
n_stim_frames=1000, n_reps_per_frame=1, shift=5)
w_fit = fit_rf_utils.compute_rf_sta(X=X, y=y)
- y_pred = fit_rf_utils.predict_linear_rf_response(w_fit.flat, X, threshold=False, dtype=np.float32)
+ y_pred = X @ w_fit.ravel()
assert np.corrcoef(y, y_pred)[0, 1] > 0.5
@@ -219,21 +155,8 @@ def test_sta_response_splits():
w_fit = fit_rf_utils.compute_rf_sta(X=x_trn, y=y_trn)
- y_pred_train = fit_rf_utils.predict_linear_rf_response(w_fit.flat, x_trn, threshold=False, dtype=np.float32)
- y_pred_test = fit_rf_utils.predict_linear_rf_response(w_fit.flat, x_tst, threshold=False, dtype=np.float32)
+ y_pred_train = x_trn @ w_fit.ravel()
+ y_pred_test = x_tst @ w_fit.ravel()
assert np.corrcoef(y_trn, y_pred_train)[0, 1] > 0.5
assert np.corrcoef(y_tst, y_pred_test)[0, 1] > 0.25
-
-
-@pytest.mark.skipif(rfest is None, reason="requires rfest")
-def test_mle_fit():
- np.random.seed(366)
-
- w_true, X, y, dt, dims = rfest.simulate.generate_data_3d_stim(
- stim_noise='pink', rf_kind='gauss', response_noise='none', design_matrix=True,
- n_stim_frames=500, n_reps_per_frame=3, shift=3)
-
- w_fit = fit_rf_utils.compute_rf_mle(X=X, y=y)
- fit_mse, random_mses = compare_fits(w_true=w_true, w_fit=w_fit)
- assert fit_mse < np.mean(random_mses) - 2 * np.std(random_mses)
diff --git a/tests/utils/test_dj_storage.py b/tests/utils/test_dj_storage.py
new file mode 100644
index 0000000..57ed132
--- /dev/null
+++ b/tests/utils/test_dj_storage.py
@@ -0,0 +1,77 @@
+from datetime import datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+import datajoint as dj
+import numpy as np
+
+from djimaging.utils.dj_storage import (
+ file_store_path,
+ load_array,
+ local_file_path,
+ local_path,
+ open_object,
+ relative_store_path,
+)
+
+
+def test_load_array_preserves_numpy_array():
+ array = np.arange(5)
+ assert load_array(array) is array
+
+
+def test_load_array_materializes_npy_ref():
+ array = np.arange(6, dtype=np.float32).reshape(1, 2, 3)
+ ref = dj.NpyRef(
+ {"shape": list(array.shape), "dtype": str(array.dtype), "path": "unused.npy"},
+ backend=object(),
+ )
+ ref._cached = array
+
+ loaded = load_array(ref)
+
+ assert loaded is array
+ np.testing.assert_array_equal(loaded.squeeze().copy().astype(np.float64), array.squeeze())
+
+
+def test_local_path_and_open_object_support_regular_files(tmp_path: Path):
+ path = tmp_path / "object.txt"
+ path.write_text("payload")
+
+ with local_path(path) as resolved:
+ assert resolved == path
+
+ with open_object(path, mode="r") as file:
+ assert file.read() == "payload"
+
+
+def test_local_file_path_resolves_file_object_ref(tmp_path: Path):
+ backend = SimpleNamespace(protocol="file", spec={"location": str(tmp_path)})
+ ref = dj.ObjectRef(
+ path="folder/object.bin",
+ size=None,
+ hash=None,
+ ext=None,
+ is_dir=False,
+ timestamp=datetime.now(),
+ _backend=backend,
+ )
+
+ assert local_file_path(ref) == tmp_path / "folder" / "object.bin"
+
+
+def test_relative_store_path_rejects_paths_outside_store(tmp_path: Path):
+ import pytest
+
+ store = tmp_path / "store"
+ store.mkdir()
+ with dj.config.override(
+ stores={"default": "test", "test": {"protocol": "file", "location": str(store)}}
+ ):
+ assert relative_store_path(store / "folder" / "file.bin", "test") == "folder/file.bin"
+ assert file_store_path("folder/file.bin", "test") == store / "folder" / "file.bin"
+ assert local_file_path("folder/file.bin", "test") == store / "folder" / "file.bin"
+ with local_path("folder/file.bin", "test") as path:
+ assert path == store / "folder" / "file.bin"
+ with pytest.raises(ValueError, match="outside DataJoint store"):
+ relative_store_path(tmp_path / "outside.bin", "test")
diff --git a/tests/utils/test_dj_utils.py b/tests/utils/test_dj_utils.py
index 771d3c8..f13dfab 100644
--- a/tests/utils/test_dj_utils.py
+++ b/tests/utils/test_dj_utils.py
@@ -1,7 +1,17 @@
+from unittest.mock import Mock
+
import numpy as np
import pytest
-from djimaging.utils.dj_utils import check_unique_one, is_equal, make_hash, merge_keys
+from djimaging.utils.dj_utils import activate_schema, check_unique_one, is_equal, make_hash, merge_keys
+
+
+def test_activate_schema_uses_explicit_name():
+ schema = Mock()
+
+ activate_schema(schema, schema_name='test_schema', create_schema=False, create_tables=False)
+
+ schema.activate.assert_called_once_with('test_schema', create_schema=False, create_tables=False)
def test_check_unique_one_int():
diff --git a/tests/utils/test_mask_utils.py b/tests/utils/test_mask_utils.py
index 88c1cb6..e40d6fe 100644
--- a/tests/utils/test_mask_utils.py
+++ b/tests/utils/test_mask_utils.py
@@ -1,6 +1,21 @@
import os
-from djimaging.utils.mask_utils import to_roi_mask_file
+import numpy as np
+
+from djimaging.utils.mask_utils import (
+ load_preferred_roi_mask_file, save_roi_mask_file, to_roi_mask_file,
+)
+
+
+def test_roi_mask_files_default_to_numpy(tmp_path):
+ data_file = str(tmp_path / 'recording.h5')
+ mask_file = to_roi_mask_file(data_file)
+ assert mask_file == str(tmp_path / 'recording_ROIs.npy')
+ mask = np.array([[0, 1], [0, 1]], dtype=np.int32)
+ save_roi_mask_file(mask_file, mask, file_format='numpy')
+ loaded, source = load_preferred_roi_mask_file([data_file])
+ np.testing.assert_array_equal(loaded, [[1, -1], [1, -1]])
+ assert source == mask_file
def test_to_roi_mask_file_unchanged():
diff --git a/uv.lock b/uv.lock
index 28cd637..626fcb9 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
version = 1
-revision = 2
+revision = 3
requires-python = ">=3.10"
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'win32'",
@@ -14,6 +14,13 @@ resolution-markers = [
"python_full_version < '3.11'",
]
+[manifest]
+constraints = [
+ { name = "aiohttp", specifier = ">=3.14.3" },
+ { name = "idna", specifier = ">=3.15" },
+ { name = "pillow", specifier = ">=12.3.0" },
+]
+
[[package]]
name = "aiohappyeyeballs"
version = "2.6.1"
@@ -25,7 +32,7 @@ wheels = [
[[package]]
name = "aiohttp"
-version = "3.13.3"
+version = "3.14.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohappyeyeballs" },
@@ -35,112 +42,129 @@ dependencies = [
{ name = "frozenlist" },
{ name = "multidict" },
{ name = "propcache" },
+ { name = "typing-extensions", marker = "python_full_version < '3.13'" },
{ name = "yarl" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/50/42/32cf8e7704ceb4481406eb87161349abb46a57fee3f008ba9cb610968646/aiohttp-3.13.3.tar.gz", hash = "sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88", size = 7844556, upload-time = "2026-01-03T17:33:05.204Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/36/d6/5aec9313ee6ea9c7cde8b891b69f4ff4001416867104580670a31daeba5b/aiohttp-3.13.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7", size = 738950, upload-time = "2026-01-03T17:29:13.002Z" },
- { url = "https://files.pythonhosted.org/packages/68/03/8fa90a7e6d11ff20a18837a8e2b5dd23db01aabc475aa9271c8ad33299f5/aiohttp-3.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821", size = 496099, upload-time = "2026-01-03T17:29:15.268Z" },
- { url = "https://files.pythonhosted.org/packages/d2/23/b81f744d402510a8366b74eb420fc0cc1170d0c43daca12d10814df85f10/aiohttp-3.13.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845", size = 491072, upload-time = "2026-01-03T17:29:16.922Z" },
- { url = "https://files.pythonhosted.org/packages/d5/e1/56d1d1c0dd334cd203dd97706ce004c1aa24b34a813b0b8daf3383039706/aiohttp-3.13.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af", size = 1671588, upload-time = "2026-01-03T17:29:18.539Z" },
- { url = "https://files.pythonhosted.org/packages/5f/34/8d7f962604f4bc2b4e39eb1220dac7d4e4cba91fb9ba0474b4ecd67db165/aiohttp-3.13.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940", size = 1640334, upload-time = "2026-01-03T17:29:21.028Z" },
- { url = "https://files.pythonhosted.org/packages/94/1d/fcccf2c668d87337ddeef9881537baee13c58d8f01f12ba8a24215f2b804/aiohttp-3.13.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160", size = 1722656, upload-time = "2026-01-03T17:29:22.531Z" },
- { url = "https://files.pythonhosted.org/packages/aa/98/c6f3b081c4c606bc1e5f2ec102e87d6411c73a9ef3616fea6f2d5c98c062/aiohttp-3.13.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7", size = 1817625, upload-time = "2026-01-03T17:29:24.276Z" },
- { url = "https://files.pythonhosted.org/packages/2c/c0/cfcc3d2e11b477f86e1af2863f3858c8850d751ce8dc39c4058a072c9e54/aiohttp-3.13.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455", size = 1672604, upload-time = "2026-01-03T17:29:26.099Z" },
- { url = "https://files.pythonhosted.org/packages/1e/77/6b4ffcbcac4c6a5d041343a756f34a6dd26174ae07f977a64fe028dda5b0/aiohttp-3.13.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279", size = 1554370, upload-time = "2026-01-03T17:29:28.121Z" },
- { url = "https://files.pythonhosted.org/packages/f2/f0/e3ddfa93f17d689dbe014ba048f18e0c9f9b456033b70e94349a2e9048be/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e", size = 1642023, upload-time = "2026-01-03T17:29:30.002Z" },
- { url = "https://files.pythonhosted.org/packages/eb/45/c14019c9ec60a8e243d06d601b33dcc4fd92379424bde3021725859d7f99/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d", size = 1649680, upload-time = "2026-01-03T17:29:31.782Z" },
- { url = "https://files.pythonhosted.org/packages/9c/fd/09c9451dae5aa5c5ed756df95ff9ef549d45d4be663bafd1e4954fd836f0/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808", size = 1692407, upload-time = "2026-01-03T17:29:33.392Z" },
- { url = "https://files.pythonhosted.org/packages/a6/81/938bc2ec33c10efd6637ccb3d22f9f3160d08e8f3aa2587a2c2d5ab578eb/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40", size = 1543047, upload-time = "2026-01-03T17:29:34.855Z" },
- { url = "https://files.pythonhosted.org/packages/f7/23/80488ee21c8d567c83045e412e1d9b7077d27171591a4eb7822586e8c06a/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29", size = 1715264, upload-time = "2026-01-03T17:29:36.389Z" },
- { url = "https://files.pythonhosted.org/packages/e2/83/259a8da6683182768200b368120ab3deff5370bed93880fb9a3a86299f34/aiohttp-3.13.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11", size = 1657275, upload-time = "2026-01-03T17:29:38.162Z" },
- { url = "https://files.pythonhosted.org/packages/3f/4f/2c41f800a0b560785c10fb316216ac058c105f9be50bdc6a285de88db625/aiohttp-3.13.3-cp310-cp310-win32.whl", hash = "sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd", size = 434053, upload-time = "2026-01-03T17:29:40.074Z" },
- { url = "https://files.pythonhosted.org/packages/80/df/29cd63c7ecfdb65ccc12f7d808cac4fa2a19544660c06c61a4a48462de0c/aiohttp-3.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c", size = 456687, upload-time = "2026-01-03T17:29:41.819Z" },
- { url = "https://files.pythonhosted.org/packages/f1/4c/a164164834f03924d9a29dc3acd9e7ee58f95857e0b467f6d04298594ebb/aiohttp-3.13.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b", size = 746051, upload-time = "2026-01-03T17:29:43.287Z" },
- { url = "https://files.pythonhosted.org/packages/82/71/d5c31390d18d4f58115037c432b7e0348c60f6f53b727cad33172144a112/aiohttp-3.13.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64", size = 499234, upload-time = "2026-01-03T17:29:44.822Z" },
- { url = "https://files.pythonhosted.org/packages/0e/c9/741f8ac91e14b1d2e7100690425a5b2b919a87a5075406582991fb7de920/aiohttp-3.13.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea", size = 494979, upload-time = "2026-01-03T17:29:46.405Z" },
- { url = "https://files.pythonhosted.org/packages/75/b5/31d4d2e802dfd59f74ed47eba48869c1c21552c586d5e81a9d0d5c2ad640/aiohttp-3.13.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a", size = 1748297, upload-time = "2026-01-03T17:29:48.083Z" },
- { url = "https://files.pythonhosted.org/packages/1a/3e/eefad0ad42959f226bb79664826883f2687d602a9ae2941a18e0484a74d3/aiohttp-3.13.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540", size = 1707172, upload-time = "2026-01-03T17:29:49.648Z" },
- { url = "https://files.pythonhosted.org/packages/c5/3a/54a64299fac2891c346cdcf2aa6803f994a2e4beeaf2e5a09dcc54acc842/aiohttp-3.13.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b", size = 1805405, upload-time = "2026-01-03T17:29:51.244Z" },
- { url = "https://files.pythonhosted.org/packages/6c/70/ddc1b7169cf64075e864f64595a14b147a895a868394a48f6a8031979038/aiohttp-3.13.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3", size = 1899449, upload-time = "2026-01-03T17:29:53.938Z" },
- { url = "https://files.pythonhosted.org/packages/a1/7e/6815aab7d3a56610891c76ef79095677b8b5be6646aaf00f69b221765021/aiohttp-3.13.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1", size = 1748444, upload-time = "2026-01-03T17:29:55.484Z" },
- { url = "https://files.pythonhosted.org/packages/6b/f2/073b145c4100da5511f457dc0f7558e99b2987cf72600d42b559db856fbc/aiohttp-3.13.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3", size = 1606038, upload-time = "2026-01-03T17:29:57.179Z" },
- { url = "https://files.pythonhosted.org/packages/0a/c1/778d011920cae03ae01424ec202c513dc69243cf2db303965615b81deeea/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440", size = 1724156, upload-time = "2026-01-03T17:29:58.914Z" },
- { url = "https://files.pythonhosted.org/packages/0e/cb/3419eabf4ec1e9ec6f242c32b689248365a1cf621891f6f0386632525494/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7", size = 1722340, upload-time = "2026-01-03T17:30:01.962Z" },
- { url = "https://files.pythonhosted.org/packages/7a/e5/76cf77bdbc435bf233c1f114edad39ed4177ccbfab7c329482b179cff4f4/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c", size = 1783041, upload-time = "2026-01-03T17:30:03.609Z" },
- { url = "https://files.pythonhosted.org/packages/9d/d4/dd1ca234c794fd29c057ce8c0566b8ef7fd6a51069de5f06fa84b9a1971c/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51", size = 1596024, upload-time = "2026-01-03T17:30:05.132Z" },
- { url = "https://files.pythonhosted.org/packages/55/58/4345b5f26661a6180afa686c473620c30a66afdf120ed3dd545bbc809e85/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4", size = 1804590, upload-time = "2026-01-03T17:30:07.135Z" },
- { url = "https://files.pythonhosted.org/packages/7b/06/05950619af6c2df7e0a431d889ba2813c9f0129cec76f663e547a5ad56f2/aiohttp-3.13.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29", size = 1740355, upload-time = "2026-01-03T17:30:09.083Z" },
- { url = "https://files.pythonhosted.org/packages/3e/80/958f16de79ba0422d7c1e284b2abd0c84bc03394fbe631d0a39ffa10e1eb/aiohttp-3.13.3-cp311-cp311-win32.whl", hash = "sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239", size = 433701, upload-time = "2026-01-03T17:30:10.869Z" },
- { url = "https://files.pythonhosted.org/packages/dc/f2/27cdf04c9851712d6c1b99df6821a6623c3c9e55956d4b1e318c337b5a48/aiohttp-3.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f", size = 457678, upload-time = "2026-01-03T17:30:12.719Z" },
- { url = "https://files.pythonhosted.org/packages/a0/be/4fc11f202955a69e0db803a12a062b8379c970c7c84f4882b6da17337cc1/aiohttp-3.13.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c", size = 739732, upload-time = "2026-01-03T17:30:14.23Z" },
- { url = "https://files.pythonhosted.org/packages/97/2c/621d5b851f94fa0bb7430d6089b3aa970a9d9b75196bc93bb624b0db237a/aiohttp-3.13.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168", size = 494293, upload-time = "2026-01-03T17:30:15.96Z" },
- { url = "https://files.pythonhosted.org/packages/5d/43/4be01406b78e1be8320bb8316dc9c42dbab553d281c40364e0f862d5661c/aiohttp-3.13.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d", size = 493533, upload-time = "2026-01-03T17:30:17.431Z" },
- { url = "https://files.pythonhosted.org/packages/8d/a8/5a35dc56a06a2c90d4742cbf35294396907027f80eea696637945a106f25/aiohttp-3.13.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29", size = 1737839, upload-time = "2026-01-03T17:30:19.422Z" },
- { url = "https://files.pythonhosted.org/packages/bf/62/4b9eeb331da56530bf2e198a297e5303e1c1ebdceeb00fe9b568a65c5a0c/aiohttp-3.13.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3", size = 1703932, upload-time = "2026-01-03T17:30:21.756Z" },
- { url = "https://files.pythonhosted.org/packages/7c/f6/af16887b5d419e6a367095994c0b1332d154f647e7dc2bd50e61876e8e3d/aiohttp-3.13.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d", size = 1771906, upload-time = "2026-01-03T17:30:23.932Z" },
- { url = "https://files.pythonhosted.org/packages/ce/83/397c634b1bcc24292fa1e0c7822800f9f6569e32934bdeef09dae7992dfb/aiohttp-3.13.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463", size = 1871020, upload-time = "2026-01-03T17:30:26Z" },
- { url = "https://files.pythonhosted.org/packages/86/f6/a62cbbf13f0ac80a70f71b1672feba90fdb21fd7abd8dbf25c0105fb6fa3/aiohttp-3.13.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc", size = 1755181, upload-time = "2026-01-03T17:30:27.554Z" },
- { url = "https://files.pythonhosted.org/packages/0a/87/20a35ad487efdd3fba93d5843efdfaa62d2f1479eaafa7453398a44faf13/aiohttp-3.13.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf", size = 1561794, upload-time = "2026-01-03T17:30:29.254Z" },
- { url = "https://files.pythonhosted.org/packages/de/95/8fd69a66682012f6716e1bc09ef8a1a2a91922c5725cb904689f112309c4/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033", size = 1697900, upload-time = "2026-01-03T17:30:31.033Z" },
- { url = "https://files.pythonhosted.org/packages/e5/66/7b94b3b5ba70e955ff597672dad1691333080e37f50280178967aff68657/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f", size = 1728239, upload-time = "2026-01-03T17:30:32.703Z" },
- { url = "https://files.pythonhosted.org/packages/47/71/6f72f77f9f7d74719692ab65a2a0252584bf8d5f301e2ecb4c0da734530a/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679", size = 1740527, upload-time = "2026-01-03T17:30:34.695Z" },
- { url = "https://files.pythonhosted.org/packages/fa/b4/75ec16cbbd5c01bdaf4a05b19e103e78d7ce1ef7c80867eb0ace42ff4488/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423", size = 1554489, upload-time = "2026-01-03T17:30:36.864Z" },
- { url = "https://files.pythonhosted.org/packages/52/8f/bc518c0eea29f8406dcf7ed1f96c9b48e3bc3995a96159b3fc11f9e08321/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce", size = 1767852, upload-time = "2026-01-03T17:30:39.433Z" },
- { url = "https://files.pythonhosted.org/packages/9d/f2/a07a75173124f31f11ea6f863dc44e6f09afe2bca45dd4e64979490deab1/aiohttp-3.13.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a", size = 1722379, upload-time = "2026-01-03T17:30:41.081Z" },
- { url = "https://files.pythonhosted.org/packages/3c/4a/1a3fee7c21350cac78e5c5cef711bac1b94feca07399f3d406972e2d8fcd/aiohttp-3.13.3-cp312-cp312-win32.whl", hash = "sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046", size = 428253, upload-time = "2026-01-03T17:30:42.644Z" },
- { url = "https://files.pythonhosted.org/packages/d9/b7/76175c7cb4eb73d91ad63c34e29fc4f77c9386bba4a65b53ba8e05ee3c39/aiohttp-3.13.3-cp312-cp312-win_amd64.whl", hash = "sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57", size = 455407, upload-time = "2026-01-03T17:30:44.195Z" },
- { url = "https://files.pythonhosted.org/packages/97/8a/12ca489246ca1faaf5432844adbfce7ff2cc4997733e0af120869345643a/aiohttp-3.13.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c", size = 734190, upload-time = "2026-01-03T17:30:45.832Z" },
- { url = "https://files.pythonhosted.org/packages/32/08/de43984c74ed1fca5c014808963cc83cb00d7bb06af228f132d33862ca76/aiohttp-3.13.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9", size = 491783, upload-time = "2026-01-03T17:30:47.466Z" },
- { url = "https://files.pythonhosted.org/packages/17/f8/8dd2cf6112a5a76f81f81a5130c57ca829d101ad583ce57f889179accdda/aiohttp-3.13.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3", size = 490704, upload-time = "2026-01-03T17:30:49.373Z" },
- { url = "https://files.pythonhosted.org/packages/6d/40/a46b03ca03936f832bc7eaa47cfbb1ad012ba1be4790122ee4f4f8cba074/aiohttp-3.13.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf", size = 1720652, upload-time = "2026-01-03T17:30:50.974Z" },
- { url = "https://files.pythonhosted.org/packages/f7/7e/917fe18e3607af92657e4285498f500dca797ff8c918bd7d90b05abf6c2a/aiohttp-3.13.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6", size = 1692014, upload-time = "2026-01-03T17:30:52.729Z" },
- { url = "https://files.pythonhosted.org/packages/71/b6/cefa4cbc00d315d68973b671cf105b21a609c12b82d52e5d0c9ae61d2a09/aiohttp-3.13.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d", size = 1759777, upload-time = "2026-01-03T17:30:54.537Z" },
- { url = "https://files.pythonhosted.org/packages/fb/e3/e06ee07b45e59e6d81498b591fc589629be1553abb2a82ce33efe2a7b068/aiohttp-3.13.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261", size = 1861276, upload-time = "2026-01-03T17:30:56.512Z" },
- { url = "https://files.pythonhosted.org/packages/7c/24/75d274228acf35ceeb2850b8ce04de9dd7355ff7a0b49d607ee60c29c518/aiohttp-3.13.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0", size = 1743131, upload-time = "2026-01-03T17:30:58.256Z" },
- { url = "https://files.pythonhosted.org/packages/04/98/3d21dde21889b17ca2eea54fdcff21b27b93f45b7bb94ca029c31ab59dc3/aiohttp-3.13.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730", size = 1556863, upload-time = "2026-01-03T17:31:00.445Z" },
- { url = "https://files.pythonhosted.org/packages/9e/84/da0c3ab1192eaf64782b03971ab4055b475d0db07b17eff925e8c93b3aa5/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91", size = 1682793, upload-time = "2026-01-03T17:31:03.024Z" },
- { url = "https://files.pythonhosted.org/packages/ff/0f/5802ada182f575afa02cbd0ec5180d7e13a402afb7c2c03a9aa5e5d49060/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3", size = 1716676, upload-time = "2026-01-03T17:31:04.842Z" },
- { url = "https://files.pythonhosted.org/packages/3f/8c/714d53bd8b5a4560667f7bbbb06b20c2382f9c7847d198370ec6526af39c/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4", size = 1733217, upload-time = "2026-01-03T17:31:06.868Z" },
- { url = "https://files.pythonhosted.org/packages/7d/79/e2176f46d2e963facea939f5be2d26368ce543622be6f00a12844d3c991f/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998", size = 1552303, upload-time = "2026-01-03T17:31:08.958Z" },
- { url = "https://files.pythonhosted.org/packages/ab/6a/28ed4dea1759916090587d1fe57087b03e6c784a642b85ef48217b0277ae/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0", size = 1763673, upload-time = "2026-01-03T17:31:10.676Z" },
- { url = "https://files.pythonhosted.org/packages/e8/35/4a3daeb8b9fab49240d21c04d50732313295e4bd813a465d840236dd0ce1/aiohttp-3.13.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591", size = 1721120, upload-time = "2026-01-03T17:31:12.575Z" },
- { url = "https://files.pythonhosted.org/packages/bc/9f/d643bb3c5fb99547323e635e251c609fbbc660d983144cfebec529e09264/aiohttp-3.13.3-cp313-cp313-win32.whl", hash = "sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf", size = 427383, upload-time = "2026-01-03T17:31:14.382Z" },
- { url = "https://files.pythonhosted.org/packages/4e/f1/ab0395f8a79933577cdd996dd2f9aa6014af9535f65dddcf88204682fe62/aiohttp-3.13.3-cp313-cp313-win_amd64.whl", hash = "sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e", size = 453899, upload-time = "2026-01-03T17:31:15.958Z" },
- { url = "https://files.pythonhosted.org/packages/99/36/5b6514a9f5d66f4e2597e40dea2e3db271e023eb7a5d22defe96ba560996/aiohttp-3.13.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808", size = 737238, upload-time = "2026-01-03T17:31:17.909Z" },
- { url = "https://files.pythonhosted.org/packages/f7/49/459327f0d5bcd8c6c9ca69e60fdeebc3622861e696490d8674a6d0cb90a6/aiohttp-3.13.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415", size = 492292, upload-time = "2026-01-03T17:31:19.919Z" },
- { url = "https://files.pythonhosted.org/packages/e8/0b/b97660c5fd05d3495b4eb27f2d0ef18dc1dc4eff7511a9bf371397ff0264/aiohttp-3.13.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f", size = 493021, upload-time = "2026-01-03T17:31:21.636Z" },
- { url = "https://files.pythonhosted.org/packages/54/d4/438efabdf74e30aeceb890c3290bbaa449780583b1270b00661126b8aae4/aiohttp-3.13.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6", size = 1717263, upload-time = "2026-01-03T17:31:23.296Z" },
- { url = "https://files.pythonhosted.org/packages/71/f2/7bddc7fd612367d1459c5bcf598a9e8f7092d6580d98de0e057eb42697ad/aiohttp-3.13.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687", size = 1669107, upload-time = "2026-01-03T17:31:25.334Z" },
- { url = "https://files.pythonhosted.org/packages/00/5a/1aeaecca40e22560f97610a329e0e5efef5e0b5afdf9f857f0d93839ab2e/aiohttp-3.13.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26", size = 1760196, upload-time = "2026-01-03T17:31:27.394Z" },
- { url = "https://files.pythonhosted.org/packages/f8/f8/0ff6992bea7bd560fc510ea1c815f87eedd745fe035589c71ce05612a19a/aiohttp-3.13.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a", size = 1843591, upload-time = "2026-01-03T17:31:29.238Z" },
- { url = "https://files.pythonhosted.org/packages/e3/d1/e30e537a15f53485b61f5be525f2157da719819e8377298502aebac45536/aiohttp-3.13.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1", size = 1720277, upload-time = "2026-01-03T17:31:31.053Z" },
- { url = "https://files.pythonhosted.org/packages/84/45/23f4c451d8192f553d38d838831ebbc156907ea6e05557f39563101b7717/aiohttp-3.13.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25", size = 1548575, upload-time = "2026-01-03T17:31:32.87Z" },
- { url = "https://files.pythonhosted.org/packages/6a/ed/0a42b127a43712eda7807e7892c083eadfaf8429ca8fb619662a530a3aab/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603", size = 1679455, upload-time = "2026-01-03T17:31:34.76Z" },
- { url = "https://files.pythonhosted.org/packages/2e/b5/c05f0c2b4b4fe2c9d55e73b6d3ed4fd6c9dc2684b1d81cbdf77e7fad9adb/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a", size = 1687417, upload-time = "2026-01-03T17:31:36.699Z" },
- { url = "https://files.pythonhosted.org/packages/c9/6b/915bc5dad66aef602b9e459b5a973529304d4e89ca86999d9d75d80cbd0b/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926", size = 1729968, upload-time = "2026-01-03T17:31:38.622Z" },
- { url = "https://files.pythonhosted.org/packages/11/3b/e84581290a9520024a08640b63d07673057aec5ca548177a82026187ba73/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba", size = 1545690, upload-time = "2026-01-03T17:31:40.57Z" },
- { url = "https://files.pythonhosted.org/packages/f5/04/0c3655a566c43fd647c81b895dfe361b9f9ad6d58c19309d45cff52d6c3b/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c", size = 1746390, upload-time = "2026-01-03T17:31:42.857Z" },
- { url = "https://files.pythonhosted.org/packages/1f/53/71165b26978f719c3419381514c9690bd5980e764a09440a10bb816ea4ab/aiohttp-3.13.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43", size = 1702188, upload-time = "2026-01-03T17:31:44.984Z" },
- { url = "https://files.pythonhosted.org/packages/29/a7/cbe6c9e8e136314fa1980da388a59d2f35f35395948a08b6747baebb6aa6/aiohttp-3.13.3-cp314-cp314-win32.whl", hash = "sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1", size = 433126, upload-time = "2026-01-03T17:31:47.463Z" },
- { url = "https://files.pythonhosted.org/packages/de/56/982704adea7d3b16614fc5936014e9af85c0e34b58f9046655817f04306e/aiohttp-3.13.3-cp314-cp314-win_amd64.whl", hash = "sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984", size = 459128, upload-time = "2026-01-03T17:31:49.2Z" },
- { url = "https://files.pythonhosted.org/packages/6c/2a/3c79b638a9c3d4658d345339d22070241ea341ed4e07b5ac60fb0f418003/aiohttp-3.13.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c", size = 769512, upload-time = "2026-01-03T17:31:51.134Z" },
- { url = "https://files.pythonhosted.org/packages/29/b9/3e5014d46c0ab0db8707e0ac2711ed28c4da0218c358a4e7c17bae0d8722/aiohttp-3.13.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592", size = 506444, upload-time = "2026-01-03T17:31:52.85Z" },
- { url = "https://files.pythonhosted.org/packages/90/03/c1d4ef9a054e151cd7839cdc497f2638f00b93cbe8043983986630d7a80c/aiohttp-3.13.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f", size = 510798, upload-time = "2026-01-03T17:31:54.91Z" },
- { url = "https://files.pythonhosted.org/packages/ea/76/8c1e5abbfe8e127c893fe7ead569148a4d5a799f7cf958d8c09f3eedf097/aiohttp-3.13.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29", size = 1868835, upload-time = "2026-01-03T17:31:56.733Z" },
- { url = "https://files.pythonhosted.org/packages/8e/ac/984c5a6f74c363b01ff97adc96a3976d9c98940b8969a1881575b279ac5d/aiohttp-3.13.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc", size = 1720486, upload-time = "2026-01-03T17:31:58.65Z" },
- { url = "https://files.pythonhosted.org/packages/b2/9a/b7039c5f099c4eb632138728828b33428585031a1e658d693d41d07d89d1/aiohttp-3.13.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2", size = 1847951, upload-time = "2026-01-03T17:32:00.989Z" },
- { url = "https://files.pythonhosted.org/packages/3c/02/3bec2b9a1ba3c19ff89a43a19324202b8eb187ca1e928d8bdac9bbdddebd/aiohttp-3.13.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587", size = 1941001, upload-time = "2026-01-03T17:32:03.122Z" },
- { url = "https://files.pythonhosted.org/packages/37/df/d879401cedeef27ac4717f6426c8c36c3091c6e9f08a9178cc87549c537f/aiohttp-3.13.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8", size = 1797246, upload-time = "2026-01-03T17:32:05.255Z" },
- { url = "https://files.pythonhosted.org/packages/8d/15/be122de1f67e6953add23335c8ece6d314ab67c8bebb3f181063010795a7/aiohttp-3.13.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632", size = 1627131, upload-time = "2026-01-03T17:32:07.607Z" },
- { url = "https://files.pythonhosted.org/packages/12/12/70eedcac9134cfa3219ab7af31ea56bc877395b1ac30d65b1bc4b27d0438/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64", size = 1795196, upload-time = "2026-01-03T17:32:09.59Z" },
- { url = "https://files.pythonhosted.org/packages/32/11/b30e1b1cd1f3054af86ebe60df96989c6a414dd87e27ad16950eee420bea/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0", size = 1782841, upload-time = "2026-01-03T17:32:11.445Z" },
- { url = "https://files.pythonhosted.org/packages/88/0d/d98a9367b38912384a17e287850f5695c528cff0f14f791ce8ee2e4f7796/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56", size = 1795193, upload-time = "2026-01-03T17:32:13.705Z" },
- { url = "https://files.pythonhosted.org/packages/43/a5/a2dfd1f5ff5581632c7f6a30e1744deda03808974f94f6534241ef60c751/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72", size = 1621979, upload-time = "2026-01-03T17:32:15.965Z" },
- { url = "https://files.pythonhosted.org/packages/fa/f0/12973c382ae7c1cccbc4417e129c5bf54c374dfb85af70893646e1f0e749/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df", size = 1822193, upload-time = "2026-01-03T17:32:18.219Z" },
- { url = "https://files.pythonhosted.org/packages/3c/5f/24155e30ba7f8c96918af1350eb0663e2430aad9e001c0489d89cd708ab1/aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa", size = 1769801, upload-time = "2026-01-03T17:32:20.25Z" },
- { url = "https://files.pythonhosted.org/packages/eb/f8/7314031ff5c10e6ece114da79b338ec17eeff3a079e53151f7e9f43c4723/aiohttp-3.13.3-cp314-cp314t-win32.whl", hash = "sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767", size = 466523, upload-time = "2026-01-03T17:32:22.215Z" },
- { url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/4d/4a99fb425c5e0cad715eea7bd190aff46f38b959a0a2dadb993705d34b26/aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b", size = 765848, upload-time = "2026-07-23T01:52:08.217Z" },
+ { url = "https://files.pythonhosted.org/packages/74/e8/43b85dc55b8e950dc644babe762add781319ea881b57b33d2cce12017d12/aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a", size = 517476, upload-time = "2026-07-23T01:52:10.846Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/9e/73b582c4dbbc3c12ef4473822475effaabf1f934b56f14f5b03fe5d3a2af/aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5", size = 515334, upload-time = "2026-07-23T01:52:12.636Z" },
+ { url = "https://files.pythonhosted.org/packages/79/03/e98c3c9e05a5bdf97defe5ff9169baba4f0ec9a901f2d60e0f060c2f051e/aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f", size = 1708830, upload-time = "2026-07-23T01:52:14.538Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/2c/26e60b694844dfd2176c57f913a22d0cd6a16f9ff202cbda7580d0328b98/aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43", size = 1674012, upload-time = "2026-07-23T01:52:16.486Z" },
+ { url = "https://files.pythonhosted.org/packages/38/65/672df92e3172cd876aacfa97a952ac560877eb169384b2991ac5b273de4c/aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9", size = 1767015, upload-time = "2026-07-23T01:52:18.28Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/c5/228dec7bfec1c373cc2217cdeb47d6456dcd7a13a4c55144930a75ae3851/aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8", size = 1858700, upload-time = "2026-07-23T01:52:20.08Z" },
+ { url = "https://files.pythonhosted.org/packages/bd/ff/cb36724e8c8d17f90ada567a9ff3efe1d6e9b549fba697a242aece180f21/aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479", size = 1714075, upload-time = "2026-07-23T01:52:22.071Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/3a/296a4135c6366376263aeef54b15caca1f07676c2ae0c525d7832f2f808a/aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b", size = 1588234, upload-time = "2026-07-23T01:52:23.757Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/81/9d5d853ef892dc066d1eb6db0e87a47348b920c1c879aa554612fdbd9d79/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d", size = 1677300, upload-time = "2026-07-23T01:52:25.861Z" },
+ { url = "https://files.pythonhosted.org/packages/68/96/021d386ae32d9b26d4b88df2e794546232ff56bb6be952bf6be227c0bbc7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d", size = 1691501, upload-time = "2026-07-23T01:52:28Z" },
+ { url = "https://files.pythonhosted.org/packages/29/9f/af66adce26a14af135c003cbd0f44ccaa68cebd30ff8ac99ca47fb4958f7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2", size = 1735113, upload-time = "2026-07-23T01:52:29.995Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/90/28c390d4c9851effe52ac25b5a2e1d92246acd00728b4fc7975dafb67484/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48", size = 1577486, upload-time = "2026-07-23T01:52:31.937Z" },
+ { url = "https://files.pythonhosted.org/packages/db/c2/00e23a1bf2abb70dd353f6987db7e7f2491d0261f7363997738c71c98f95/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f", size = 1751353, upload-time = "2026-07-23T01:52:33.688Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/7d/d51a706a8cbfa57f0611127daf61ab3ae02ab8420b0407412079227d1c65/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32", size = 1698681, upload-time = "2026-07-23T01:52:38.167Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/b0/90bd5cd9fdd9787cb4211d284d1fb8401339a933cb0227a15b71e789232f/aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e", size = 456733, upload-time = "2026-07-23T01:52:41.823Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/15/fe5b8f6a71ae112bc677163d0b0701bda5dc15005249582258ede0eb88c7/aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c", size = 480460, upload-time = "2026-07-23T01:52:43.905Z" },
+ { url = "https://files.pythonhosted.org/packages/54/00/45e98b6645cd7f00a4b78b749ebd309094b0eaeb2d2e96157eadbc0d0050/aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb", size = 453479, upload-time = "2026-07-23T01:52:46.075Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/5c/b3e4ff8ad43a8afef9602c5e90285936da1beaea8b029016b793891f03c3/aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3", size = 764250, upload-time = "2026-07-23T01:52:48.525Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/da/f1b384465e51449d844056b75070461da03a9a23e6c1747003695bf4172a/aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a", size = 516281, upload-time = "2026-07-23T01:52:51.047Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/3f/01264f820ee2e3712a827892b1cd6ff80f3300c1fcbffbb45714a915d47a/aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8", size = 514742, upload-time = "2026-07-23T01:52:53.779Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/8d/a71c6f2db52ac1ed142b133f7feddaa6b70539c3f4de24d7e226c95b794c/aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239", size = 1780613, upload-time = "2026-07-23T01:52:56.948Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/11/3dd9b3fb3a170f6ec9011b5291d876a6fab4086714c9e158600edf01b4fd/aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f", size = 1737688, upload-time = "2026-07-23T01:52:59.294Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/3e/834c26918be7d88068822b40e0db30fca50b5f4fe79104aa16a93f1d74e6/aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06", size = 1845742, upload-time = "2026-07-23T01:53:01.641Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/c9/49ab8572df7d66bc13d11e31f781292badb04180dd87ba98733066c6aed7/aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929", size = 1928412, upload-time = "2026-07-23T01:53:04.018Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/b9/2b8f0c0ce09c87a1daf80fd483431b56b1435d3f62789bc86f572e1245de/aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db", size = 1786220, upload-time = "2026-07-23T01:53:06.481Z" },
+ { url = "https://files.pythonhosted.org/packages/85/00/9c45f81de11710460edfa1dc81317b6e882703b160926c879a9d20da9fcc/aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce", size = 1637231, upload-time = "2026-07-23T01:53:10.258Z" },
+ { url = "https://files.pythonhosted.org/packages/19/ce/967d628e910756f3539c6107cb7844a1b69440dcb3029a5ee7871b09ab63/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c", size = 1753161, upload-time = "2026-07-23T01:53:13.817Z" },
+ { url = "https://files.pythonhosted.org/packages/11/b2/0c3d4114f0aee4f580f5b3b4eb71b24d7a23b834ea506a4dfebe76513f35/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15", size = 1756356, upload-time = "2026-07-23T01:53:16.211Z" },
+ { url = "https://files.pythonhosted.org/packages/63/5d/99e7d91c82f1399d1ae2a854e080bd1493fbc31e5e959dbc4ec33dac3bec/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c", size = 1819846, upload-time = "2026-07-23T01:53:18.289Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/05/d5e1cb6480eeffd3f901d40a2c5e2d1e7effdc797837da3b490272699f13/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae", size = 1628531, upload-time = "2026-07-23T01:53:23.86Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/90/b934682bcaefae18a9e04f3dff5b68522ba810906358ae5029b68110ea3b/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910", size = 1832712, upload-time = "2026-07-23T01:53:27.551Z" },
+ { url = "https://files.pythonhosted.org/packages/21/df/6061679faaf81fac746e7307c7adb71e858071a5d34c27583afefc64f543/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7", size = 1775014, upload-time = "2026-07-23T01:53:30.223Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/1d/f854878bbc69b88faefe924b619a34a6f59ec05fd387c77690667eaa75eb/aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa", size = 456006, upload-time = "2026-07-23T01:53:34.97Z" },
+ { url = "https://files.pythonhosted.org/packages/73/0c/2af9d1674baccd1dbd47282a93d660a22e57ef6167c856deb24b4214fbab/aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d", size = 481069, upload-time = "2026-07-23T01:53:39.673Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/76/88401ff3fc95e85c5fc38d588f36f55e61ecb64343b2bc8d69326f453cc0/aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39", size = 453021, upload-time = "2026-07-23T01:53:43.749Z" },
+ { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" },
+ { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" },
+ { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" },
+ { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" },
+ { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" },
+ { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" },
+ { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" },
+ { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" },
+ { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" },
+ { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" },
+ { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" },
+ { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" },
+ { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" },
+ { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" },
+ { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" },
+ { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" },
+ { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" },
+ { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" },
+ { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" },
+ { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" },
+ { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" },
+ { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" },
+ { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" },
+ { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" },
+ { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" },
+ { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" },
+ { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" },
+ { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" },
+ { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" },
+ { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" },
+ { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" },
+ { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" },
+ { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" },
+ { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" },
+ { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" },
+ { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" },
+ { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" },
+ { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" },
+ { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" },
+ { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" },
+ { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" },
+ { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" },
+ { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" },
+ { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" },
+ { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" },
]
[[package]]
@@ -281,15 +305,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" },
]
-[[package]]
-name = "cached-property"
-version = "2.0.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/76/4b/3d870836119dbe9a5e3c9a61af8cc1a8b69d75aea564572e385882d5aefb/cached_property-2.0.1.tar.gz", hash = "sha256:484d617105e3ee0e4f1f58725e72a8ef9e93deee462222dbd51cd91230897641", size = 10574, upload-time = "2024-10-25T15:43:55.667Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/11/0e/7d8225aab3bc1a0f5811f8e1b557aa034ac04bdf641925b30d3caf586b28/cached_property-2.0.1-py3-none-any.whl", hash = "sha256:f617d70ab1100b7bcf6e42228f9ddcb78c676ffa167278d9f730d1c2fba69ccb", size = 7428, upload-time = "2024-10-25T15:43:54.711Z" },
-]
-
[[package]]
name = "colorama"
version = "0.4.6"
@@ -507,7 +522,7 @@ wheels = [
[[package]]
name = "datajoint"
-version = "2.1.1"
+version = "2.3.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "deepdiff" },
@@ -516,6 +531,7 @@ dependencies = [
{ name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+ { name = "packaging" },
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "pydantic-settings" },
@@ -524,9 +540,9 @@ dependencies = [
{ name = "pyparsing" },
{ name = "tqdm" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/62/52/b341a41abae666f5c9bbbdfb08fb4836681acb8cddb562e7482140820005/datajoint-2.1.1.tar.gz", hash = "sha256:db2965db3a5d82a7c01084ea5cc773867e421288a37feb227affee37b2a40702", size = 1708931, upload-time = "2026-02-17T19:56:37.894Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/62/9c/f5f71f241ae31b8861f40c4fcbcc5b0f607c5f3b836dbd834e18643553d1/datajoint-2.3.2.tar.gz", hash = "sha256:47efee045b6eb346353132d4bcd41c58583b661ef880831357fc30227557f3fa", size = 678052, upload-time = "2026-07-21T21:31:40.867Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/ec/26/db3b1edf7176ee55329606324cb36b861f0106f456dc94a096fc2fdf1826/datajoint-2.1.1-py3-none-any.whl", hash = "sha256:8c4c8e052958e70aba998a1b1f46fd1266ff0d2946b19966992fb3a4e40e4d94", size = 221980, upload-time = "2026-02-17T19:56:36.442Z" },
+ { url = "https://files.pythonhosted.org/packages/6f/09/4ffdd209ef5bbf7e086c70c633e5fece743a5a6e55e14d0d28422251943c/datajoint-2.3.2-py3-none-any.whl", hash = "sha256:734c74465faef9d1e3053bc22e5aa3a47f82117752c90af00a26009af80ec2c6", size = 252200, upload-time = "2026-07-21T21:31:39.593Z" },
]
[[package]]
@@ -557,7 +573,6 @@ source = { editable = "." }
dependencies = [
{ name = "astropy", version = "6.1.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "astropy", version = "7.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
- { name = "cached-property" },
{ name = "datajoint" },
{ name = "h5py" },
{ name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
@@ -594,26 +609,25 @@ rf = [
[package.metadata]
requires-dist = [
- { name = "astropy" },
- { name = "cached-property" },
- { name = "datajoint", specifier = ">=0.13" },
- { name = "h5py" },
- { name = "ipycanvas", marker = "extra == 'autorois'" },
- { name = "ipython" },
- { name = "ipywidgets", marker = "extra == 'autorois'" },
- { name = "matplotlib" },
- { name = "numba" },
- { name = "numpy" },
- { name = "opencv-python" },
- { name = "pandas" },
- { name = "pytest", marker = "extra == 'dev'" },
- { name = "pytorch-lightning", marker = "extra == 'autorois'" },
+ { name = "astropy", specifier = ">=6.1.0" },
+ { name = "datajoint", specifier = ">=2.3.1,<2.4" },
+ { name = "h5py", specifier = ">=3.11.0" },
+ { name = "ipycanvas", marker = "extra == 'autorois'", specifier = ">=0.13.0" },
+ { name = "ipython", specifier = ">=8.12.3" },
+ { name = "ipywidgets", marker = "extra == 'autorois'", specifier = ">=8.0.0" },
+ { name = "matplotlib", specifier = ">=3.8.4" },
+ { name = "numba", specifier = ">=0.60.0" },
+ { name = "numpy", specifier = ">=2.0.0" },
+ { name = "opencv-python", specifier = ">=4.10.0.84" },
+ { name = "pandas", specifier = ">=2.2.2" },
+ { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.3" },
+ { name = "pytorch-lightning", marker = "extra == 'autorois'", specifier = ">=2.4.0" },
{ name = "rfest", marker = "extra == 'rf'", specifier = ">=2.1.2" },
{ name = "scanm-support", git = "https://github.com/eulerlab/ScanM_support.git" },
- { name = "scikit-learn" },
- { name = "scipy" },
- { name = "seaborn" },
- { name = "torch", marker = "extra == 'autorois'" },
+ { name = "scikit-learn", specifier = ">=1.7.2" },
+ { name = "scipy", specifier = ">=1.13.0" },
+ { name = "seaborn", specifier = ">=0.13.2" },
+ { name = "torch", marker = "extra == 'autorois'", specifier = ">=2.4.1" },
]
provides-extras = ["autorois", "rf", "dev"]
@@ -900,11 +914,11 @@ wheels = [
[[package]]
name = "idna"
-version = "3.11"
+version = "3.15"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" },
]
[[package]]
@@ -2108,7 +2122,7 @@ name = "pexpect"
version = "4.9.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "ptyprocess" },
+ { name = "ptyprocess", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" }
wheels = [
@@ -2117,100 +2131,96 @@ wheels = [
[[package]]
name = "pillow"
-version = "12.1.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/1f/42/5c74462b4fd957fcd7b13b04fb3205ff8349236ea74c7c375766d6c82288/pillow-12.1.1.tar.gz", hash = "sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4", size = 46980264, upload-time = "2026-02-11T04:23:07.146Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/1d/30/5bd3d794762481f8c8ae9c80e7b76ecea73b916959eb587521358ef0b2f9/pillow-12.1.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f1625b72740fdda5d77b4def688eb8fd6490975d06b909fd19f13f391e077e0", size = 5304099, upload-time = "2026-02-11T04:20:06.13Z" },
- { url = "https://files.pythonhosted.org/packages/bd/c1/aab9e8f3eeb4490180e357955e15c2ef74b31f64790ff356c06fb6cf6d84/pillow-12.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:178aa072084bd88ec759052feca8e56cbb14a60b39322b99a049e58090479713", size = 4657880, upload-time = "2026-02-11T04:20:09.291Z" },
- { url = "https://files.pythonhosted.org/packages/f1/0a/9879e30d56815ad529d3985aeff5af4964202425c27261a6ada10f7cbf53/pillow-12.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b66e95d05ba806247aaa1561f080abc7975daf715c30780ff92a20e4ec546e1b", size = 6222587, upload-time = "2026-02-11T04:20:10.82Z" },
- { url = "https://files.pythonhosted.org/packages/5a/5f/a1b72ff7139e4f89014e8d451442c74a774d5c43cd938fb0a9f878576b37/pillow-12.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:89c7e895002bbe49cdc5426150377cbbc04767d7547ed145473f496dfa40408b", size = 8027678, upload-time = "2026-02-11T04:20:12.455Z" },
- { url = "https://files.pythonhosted.org/packages/e2/c2/c7cb187dac79a3d22c3ebeae727abee01e077c8c7d930791dc592f335153/pillow-12.1.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a5cbdcddad0af3da87cb16b60d23648bc3b51967eb07223e9fed77a82b457c4", size = 6335777, upload-time = "2026-02-11T04:20:14.441Z" },
- { url = "https://files.pythonhosted.org/packages/0c/7b/f9b09a7804ec7336effb96c26d37c29d27225783dc1501b7d62dcef6ae25/pillow-12.1.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f51079765661884a486727f0729d29054242f74b46186026582b4e4769918e4", size = 7027140, upload-time = "2026-02-11T04:20:16.387Z" },
- { url = "https://files.pythonhosted.org/packages/98/b2/2fa3c391550bd421b10849d1a2144c44abcd966daadd2f7c12e19ea988c4/pillow-12.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:99c1506ea77c11531d75e3a412832a13a71c7ebc8192ab9e4b2e355555920e3e", size = 6449855, upload-time = "2026-02-11T04:20:18.554Z" },
- { url = "https://files.pythonhosted.org/packages/96/ff/9caf4b5b950c669263c39e96c78c0d74a342c71c4f43fd031bb5cb7ceac9/pillow-12.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:36341d06738a9f66c8287cf8b876d24b18db9bd8740fa0672c74e259ad408cff", size = 7151329, upload-time = "2026-02-11T04:20:20.646Z" },
- { url = "https://files.pythonhosted.org/packages/7b/f8/4b24841f582704da675ca535935bccb32b00a6da1226820845fac4a71136/pillow-12.1.1-cp310-cp310-win32.whl", hash = "sha256:6c52f062424c523d6c4db85518774cc3d50f5539dd6eed32b8f6229b26f24d40", size = 6325574, upload-time = "2026-02-11T04:20:22.43Z" },
- { url = "https://files.pythonhosted.org/packages/f8/f9/9f6b01c0881d7036063aa6612ef04c0e2cad96be21325a1e92d0203f8e91/pillow-12.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:c6008de247150668a705a6338156efb92334113421ceecf7438a12c9a12dab23", size = 7032347, upload-time = "2026-02-11T04:20:23.932Z" },
- { url = "https://files.pythonhosted.org/packages/79/13/c7922edded3dcdaf10c59297540b72785620abc0538872c819915746757d/pillow-12.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:1a9b0ee305220b392e1124a764ee4265bd063e54a751a6b62eff69992f457fa9", size = 2453457, upload-time = "2026-02-11T04:20:25.392Z" },
- { url = "https://files.pythonhosted.org/packages/2b/46/5da1ec4a5171ee7bf1a0efa064aba70ba3d6e0788ce3f5acd1375d23c8c0/pillow-12.1.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e879bb6cd5c73848ef3b2b48b8af9ff08c5b71ecda8048b7dd22d8a33f60be32", size = 5304084, upload-time = "2026-02-11T04:20:27.501Z" },
- { url = "https://files.pythonhosted.org/packages/78/93/a29e9bc02d1cf557a834da780ceccd54e02421627200696fcf805ebdc3fb/pillow-12.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:365b10bb9417dd4498c0e3b128018c4a624dc11c7b97d8cc54effe3b096f4c38", size = 4657866, upload-time = "2026-02-11T04:20:29.827Z" },
- { url = "https://files.pythonhosted.org/packages/13/84/583a4558d492a179d31e4aae32eadce94b9acf49c0337c4ce0b70e0a01f2/pillow-12.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d4ce8e329c93845720cd2014659ca67eac35f6433fd3050393d85f3ecef0dad5", size = 6232148, upload-time = "2026-02-11T04:20:31.329Z" },
- { url = "https://files.pythonhosted.org/packages/d5/e2/53c43334bbbb2d3b938978532fbda8e62bb6e0b23a26ce8592f36bcc4987/pillow-12.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc354a04072b765eccf2204f588a7a532c9511e8b9c7f900e1b64e3e33487090", size = 8038007, upload-time = "2026-02-11T04:20:34.225Z" },
- { url = "https://files.pythonhosted.org/packages/b8/a6/3d0e79c8a9d58150dd98e199d7c1c56861027f3829a3a60b3c2784190180/pillow-12.1.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e7976bf1910a8116b523b9f9f58bf410f3e8aa330cd9a2bb2953f9266ab49af", size = 6345418, upload-time = "2026-02-11T04:20:35.858Z" },
- { url = "https://files.pythonhosted.org/packages/a2/c8/46dfeac5825e600579157eea177be43e2f7ff4a99da9d0d0a49533509ac5/pillow-12.1.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:597bd9c8419bc7c6af5604e55847789b69123bbe25d65cc6ad3012b4f3c98d8b", size = 7034590, upload-time = "2026-02-11T04:20:37.91Z" },
- { url = "https://files.pythonhosted.org/packages/af/bf/e6f65d3db8a8bbfeaf9e13cc0417813f6319863a73de934f14b2229ada18/pillow-12.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2c1fc0f2ca5f96a3c8407e41cca26a16e46b21060fe6d5b099d2cb01412222f5", size = 6458655, upload-time = "2026-02-11T04:20:39.496Z" },
- { url = "https://files.pythonhosted.org/packages/f9/c2/66091f3f34a25894ca129362e510b956ef26f8fb67a0e6417bc5744e56f1/pillow-12.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:578510d88c6229d735855e1f278aa305270438d36a05031dfaae5067cc8eb04d", size = 7159286, upload-time = "2026-02-11T04:20:41.139Z" },
- { url = "https://files.pythonhosted.org/packages/7b/5a/24bc8eb526a22f957d0cec6243146744966d40857e3d8deb68f7902ca6c1/pillow-12.1.1-cp311-cp311-win32.whl", hash = "sha256:7311c0a0dcadb89b36b7025dfd8326ecfa36964e29913074d47382706e516a7c", size = 6328663, upload-time = "2026-02-11T04:20:43.184Z" },
- { url = "https://files.pythonhosted.org/packages/31/03/bef822e4f2d8f9d7448c133d0a18185d3cce3e70472774fffefe8b0ed562/pillow-12.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:fbfa2a7c10cc2623f412753cddf391c7f971c52ca40a3f65dc5039b2939e8563", size = 7031448, upload-time = "2026-02-11T04:20:44.696Z" },
- { url = "https://files.pythonhosted.org/packages/49/70/f76296f53610bd17b2e7d31728b8b7825e3ac3b5b3688b51f52eab7c0818/pillow-12.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:b81b5e3511211631b3f672a595e3221252c90af017e399056d0faabb9538aa80", size = 2453651, upload-time = "2026-02-11T04:20:46.243Z" },
- { url = "https://files.pythonhosted.org/packages/07/d3/8df65da0d4df36b094351dce696f2989bec731d4f10e743b1c5f4da4d3bf/pillow-12.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab323b787d6e18b3d91a72fc99b1a2c28651e4358749842b8f8dfacd28ef2052", size = 5262803, upload-time = "2026-02-11T04:20:47.653Z" },
- { url = "https://files.pythonhosted.org/packages/d6/71/5026395b290ff404b836e636f51d7297e6c83beceaa87c592718747e670f/pillow-12.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:adebb5bee0f0af4909c30db0d890c773d1a92ffe83da908e2e9e720f8edf3984", size = 4657601, upload-time = "2026-02-11T04:20:49.328Z" },
- { url = "https://files.pythonhosted.org/packages/b1/2e/1001613d941c67442f745aff0f7cc66dd8df9a9c084eb497e6a543ee6f7e/pillow-12.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb66b7cc26f50977108790e2456b7921e773f23db5630261102233eb355a3b79", size = 6234995, upload-time = "2026-02-11T04:20:51.032Z" },
- { url = "https://files.pythonhosted.org/packages/07/26/246ab11455b2549b9233dbd44d358d033a2f780fa9007b61a913c5b2d24e/pillow-12.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee2810642b2898bb187ced9b349e95d2a7272930796e022efaf12e99dccd293", size = 8045012, upload-time = "2026-02-11T04:20:52.882Z" },
- { url = "https://files.pythonhosted.org/packages/b2/8b/07587069c27be7535ac1fe33874e32de118fbd34e2a73b7f83436a88368c/pillow-12.1.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0b1cd6232e2b618adcc54d9882e4e662a089d5768cd188f7c245b4c8c44a397", size = 6349638, upload-time = "2026-02-11T04:20:54.444Z" },
- { url = "https://files.pythonhosted.org/packages/ff/79/6df7b2ee763d619cda2fb4fea498e5f79d984dae304d45a8999b80d6cf5c/pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7aac39bcf8d4770d089588a2e1dd111cbaa42df5a94be3114222057d68336bd0", size = 7041540, upload-time = "2026-02-11T04:20:55.97Z" },
- { url = "https://files.pythonhosted.org/packages/2c/5e/2ba19e7e7236d7529f4d873bdaf317a318896bac289abebd4bb00ef247f0/pillow-12.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ab174cd7d29a62dd139c44bf74b698039328f45cb03b4596c43473a46656b2f3", size = 6462613, upload-time = "2026-02-11T04:20:57.542Z" },
- { url = "https://files.pythonhosted.org/packages/03/03/31216ec124bb5c3dacd74ce8efff4cc7f52643653bad4825f8f08c697743/pillow-12.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:339ffdcb7cbeaa08221cd401d517d4b1fe7a9ed5d400e4a8039719238620ca35", size = 7166745, upload-time = "2026-02-11T04:20:59.196Z" },
- { url = "https://files.pythonhosted.org/packages/1f/e7/7c4552d80052337eb28653b617eafdef39adfb137c49dd7e831b8dc13bc5/pillow-12.1.1-cp312-cp312-win32.whl", hash = "sha256:5d1f9575a12bed9e9eedd9a4972834b08c97a352bd17955ccdebfeca5913fa0a", size = 6328823, upload-time = "2026-02-11T04:21:01.385Z" },
- { url = "https://files.pythonhosted.org/packages/3d/17/688626d192d7261bbbf98846fc98995726bddc2c945344b65bec3a29d731/pillow-12.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:21329ec8c96c6e979cd0dfd29406c40c1d52521a90544463057d2aaa937d66a6", size = 7033367, upload-time = "2026-02-11T04:21:03.536Z" },
- { url = "https://files.pythonhosted.org/packages/ed/fe/a0ef1f73f939b0eca03ee2c108d0043a87468664770612602c63266a43c4/pillow-12.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:af9a332e572978f0218686636610555ae3defd1633597be015ed50289a03c523", size = 2453811, upload-time = "2026-02-11T04:21:05.116Z" },
- { url = "https://files.pythonhosted.org/packages/d5/11/6db24d4bd7685583caeae54b7009584e38da3c3d4488ed4cd25b439de486/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e", size = 4062689, upload-time = "2026-02-11T04:21:06.804Z" },
- { url = "https://files.pythonhosted.org/packages/33/c0/ce6d3b1fe190f0021203e0d9b5b99e57843e345f15f9ef22fcd43842fd21/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9", size = 4138535, upload-time = "2026-02-11T04:21:08.452Z" },
- { url = "https://files.pythonhosted.org/packages/a0/c6/d5eb6a4fb32a3f9c21a8c7613ec706534ea1cf9f4b3663e99f0d83f6fca8/pillow-12.1.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6", size = 3601364, upload-time = "2026-02-11T04:21:10.194Z" },
- { url = "https://files.pythonhosted.org/packages/14/a1/16c4b823838ba4c9c52c0e6bbda903a3fe5a1bdbf1b8eb4fff7156f3e318/pillow-12.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60", size = 5262561, upload-time = "2026-02-11T04:21:11.742Z" },
- { url = "https://files.pythonhosted.org/packages/bb/ad/ad9dc98ff24f485008aa5cdedaf1a219876f6f6c42a4626c08bc4e80b120/pillow-12.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2", size = 4657460, upload-time = "2026-02-11T04:21:13.786Z" },
- { url = "https://files.pythonhosted.org/packages/9e/1b/f1a4ea9a895b5732152789326202a82464d5254759fbacae4deea3069334/pillow-12.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850", size = 6232698, upload-time = "2026-02-11T04:21:15.949Z" },
- { url = "https://files.pythonhosted.org/packages/95/f4/86f51b8745070daf21fd2e5b1fe0eb35d4db9ca26e6d58366562fb56a743/pillow-12.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289", size = 8041706, upload-time = "2026-02-11T04:21:17.723Z" },
- { url = "https://files.pythonhosted.org/packages/29/9b/d6ecd956bb1266dd1045e995cce9b8d77759e740953a1c9aad9502a0461e/pillow-12.1.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e", size = 6346621, upload-time = "2026-02-11T04:21:19.547Z" },
- { url = "https://files.pythonhosted.org/packages/71/24/538bff45bde96535d7d998c6fed1a751c75ac7c53c37c90dc2601b243893/pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717", size = 7038069, upload-time = "2026-02-11T04:21:21.378Z" },
- { url = "https://files.pythonhosted.org/packages/94/0e/58cb1a6bc48f746bc4cb3adb8cabff73e2742c92b3bf7a220b7cf69b9177/pillow-12.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a", size = 6460040, upload-time = "2026-02-11T04:21:23.148Z" },
- { url = "https://files.pythonhosted.org/packages/6c/57/9045cb3ff11eeb6c1adce3b2d60d7d299d7b273a2e6c8381a524abfdc474/pillow-12.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029", size = 7164523, upload-time = "2026-02-11T04:21:25.01Z" },
- { url = "https://files.pythonhosted.org/packages/73/f2/9be9cb99f2175f0d4dbadd6616ce1bf068ee54a28277ea1bf1fbf729c250/pillow-12.1.1-cp313-cp313-win32.whl", hash = "sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b", size = 6332552, upload-time = "2026-02-11T04:21:27.238Z" },
- { url = "https://files.pythonhosted.org/packages/3f/eb/b0834ad8b583d7d9d42b80becff092082a1c3c156bb582590fcc973f1c7c/pillow-12.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1", size = 7040108, upload-time = "2026-02-11T04:21:29.462Z" },
- { url = "https://files.pythonhosted.org/packages/d5/7d/fc09634e2aabdd0feabaff4a32f4a7d97789223e7c2042fd805ea4b4d2c2/pillow-12.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a", size = 2453712, upload-time = "2026-02-11T04:21:31.072Z" },
- { url = "https://files.pythonhosted.org/packages/19/2a/b9d62794fc8a0dd14c1943df68347badbd5511103e0d04c035ffe5cf2255/pillow-12.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da", size = 5264880, upload-time = "2026-02-11T04:21:32.865Z" },
- { url = "https://files.pythonhosted.org/packages/26/9d/e03d857d1347fa5ed9247e123fcd2a97b6220e15e9cb73ca0a8d91702c6e/pillow-12.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc", size = 4660616, upload-time = "2026-02-11T04:21:34.97Z" },
- { url = "https://files.pythonhosted.org/packages/f7/ec/8a6d22afd02570d30954e043f09c32772bfe143ba9285e2fdb11284952cd/pillow-12.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c", size = 6269008, upload-time = "2026-02-11T04:21:36.623Z" },
- { url = "https://files.pythonhosted.org/packages/3d/1d/6d875422c9f28a4a361f495a5f68d9de4a66941dc2c619103ca335fa6446/pillow-12.1.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8", size = 8073226, upload-time = "2026-02-11T04:21:38.585Z" },
- { url = "https://files.pythonhosted.org/packages/a1/cd/134b0b6ee5eda6dc09e25e24b40fdafe11a520bc725c1d0bbaa5e00bf95b/pillow-12.1.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20", size = 6380136, upload-time = "2026-02-11T04:21:40.562Z" },
- { url = "https://files.pythonhosted.org/packages/7a/a9/7628f013f18f001c1b98d8fffe3452f306a70dc6aba7d931019e0492f45e/pillow-12.1.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13", size = 7067129, upload-time = "2026-02-11T04:21:42.521Z" },
- { url = "https://files.pythonhosted.org/packages/1e/f8/66ab30a2193b277785601e82ee2d49f68ea575d9637e5e234faaa98efa4c/pillow-12.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf", size = 6491807, upload-time = "2026-02-11T04:21:44.22Z" },
- { url = "https://files.pythonhosted.org/packages/da/0b/a877a6627dc8318fdb84e357c5e1a758c0941ab1ddffdafd231983788579/pillow-12.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524", size = 7190954, upload-time = "2026-02-11T04:21:46.114Z" },
- { url = "https://files.pythonhosted.org/packages/83/43/6f732ff85743cf746b1361b91665d9f5155e1483817f693f8d57ea93147f/pillow-12.1.1-cp313-cp313t-win32.whl", hash = "sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986", size = 6336441, upload-time = "2026-02-11T04:21:48.22Z" },
- { url = "https://files.pythonhosted.org/packages/3b/44/e865ef3986611bb75bfabdf94a590016ea327833f434558801122979cd0e/pillow-12.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c", size = 7045383, upload-time = "2026-02-11T04:21:50.015Z" },
- { url = "https://files.pythonhosted.org/packages/a8/c6/f4fb24268d0c6908b9f04143697ea18b0379490cb74ba9e8d41b898bd005/pillow-12.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3", size = 2456104, upload-time = "2026-02-11T04:21:51.633Z" },
- { url = "https://files.pythonhosted.org/packages/03/d0/bebb3ffbf31c5a8e97241476c4cf8b9828954693ce6744b4a2326af3e16b/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af", size = 4062652, upload-time = "2026-02-11T04:21:53.19Z" },
- { url = "https://files.pythonhosted.org/packages/2d/c0/0e16fb0addda4851445c28f8350d8c512f09de27bbb0d6d0bbf8b6709605/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f", size = 4138823, upload-time = "2026-02-11T04:22:03.088Z" },
- { url = "https://files.pythonhosted.org/packages/6b/fb/6170ec655d6f6bb6630a013dd7cf7bc218423d7b5fa9071bf63dc32175ae/pillow-12.1.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642", size = 3601143, upload-time = "2026-02-11T04:22:04.909Z" },
- { url = "https://files.pythonhosted.org/packages/59/04/dc5c3f297510ba9a6837cbb318b87dd2b8f73eb41a43cc63767f65cb599c/pillow-12.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd", size = 5266254, upload-time = "2026-02-11T04:22:07.656Z" },
- { url = "https://files.pythonhosted.org/packages/05/30/5db1236b0d6313f03ebf97f5e17cda9ca060f524b2fcc875149a8360b21c/pillow-12.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202", size = 4657499, upload-time = "2026-02-11T04:22:09.613Z" },
- { url = "https://files.pythonhosted.org/packages/6f/18/008d2ca0eb612e81968e8be0bbae5051efba24d52debf930126d7eaacbba/pillow-12.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f", size = 6232137, upload-time = "2026-02-11T04:22:11.434Z" },
- { url = "https://files.pythonhosted.org/packages/70/f1/f14d5b8eeb4b2cd62b9f9f847eb6605f103df89ef619ac68f92f748614ea/pillow-12.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f", size = 8042721, upload-time = "2026-02-11T04:22:13.321Z" },
- { url = "https://files.pythonhosted.org/packages/5a/d6/17824509146e4babbdabf04d8171491fa9d776f7061ff6e727522df9bd03/pillow-12.1.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f", size = 6347798, upload-time = "2026-02-11T04:22:15.449Z" },
- { url = "https://files.pythonhosted.org/packages/d1/ee/c85a38a9ab92037a75615aba572c85ea51e605265036e00c5b67dfafbfe2/pillow-12.1.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e", size = 7039315, upload-time = "2026-02-11T04:22:17.24Z" },
- { url = "https://files.pythonhosted.org/packages/ec/f3/bc8ccc6e08a148290d7523bde4d9a0d6c981db34631390dc6e6ec34cacf6/pillow-12.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0", size = 6462360, upload-time = "2026-02-11T04:22:19.111Z" },
- { url = "https://files.pythonhosted.org/packages/f6/ab/69a42656adb1d0665ab051eec58a41f169ad295cf81ad45406963105408f/pillow-12.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb", size = 7165438, upload-time = "2026-02-11T04:22:21.041Z" },
- { url = "https://files.pythonhosted.org/packages/02/46/81f7aa8941873f0f01d4b55cc543b0a3d03ec2ee30d617a0448bf6bd6dec/pillow-12.1.1-cp314-cp314-win32.whl", hash = "sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f", size = 6431503, upload-time = "2026-02-11T04:22:22.833Z" },
- { url = "https://files.pythonhosted.org/packages/40/72/4c245f7d1044b67affc7f134a09ea619d4895333d35322b775b928180044/pillow-12.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15", size = 7176748, upload-time = "2026-02-11T04:22:24.64Z" },
- { url = "https://files.pythonhosted.org/packages/e4/ad/8a87bdbe038c5c698736e3348af5c2194ffb872ea52f11894c95f9305435/pillow-12.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f", size = 2544314, upload-time = "2026-02-11T04:22:26.685Z" },
- { url = "https://files.pythonhosted.org/packages/6c/9d/efd18493f9de13b87ede7c47e69184b9e859e4427225ea962e32e56a49bc/pillow-12.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8", size = 5268612, upload-time = "2026-02-11T04:22:29.884Z" },
- { url = "https://files.pythonhosted.org/packages/f8/f1/4f42eb2b388eb2ffc660dcb7f7b556c1015c53ebd5f7f754965ef997585b/pillow-12.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9", size = 4660567, upload-time = "2026-02-11T04:22:31.799Z" },
- { url = "https://files.pythonhosted.org/packages/01/54/df6ef130fa43e4b82e32624a7b821a2be1c5653a5fdad8469687a7db4e00/pillow-12.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60", size = 6269951, upload-time = "2026-02-11T04:22:33.921Z" },
- { url = "https://files.pythonhosted.org/packages/a9/48/618752d06cc44bb4aae8ce0cd4e6426871929ed7b46215638088270d9b34/pillow-12.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7", size = 8074769, upload-time = "2026-02-11T04:22:35.877Z" },
- { url = "https://files.pythonhosted.org/packages/c3/bd/f1d71eb39a72fa088d938655afba3e00b38018d052752f435838961127d8/pillow-12.1.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f", size = 6381358, upload-time = "2026-02-11T04:22:37.698Z" },
- { url = "https://files.pythonhosted.org/packages/64/ef/c784e20b96674ed36a5af839305f55616f8b4f8aa8eeccf8531a6e312243/pillow-12.1.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586", size = 7068558, upload-time = "2026-02-11T04:22:39.597Z" },
- { url = "https://files.pythonhosted.org/packages/73/cb/8059688b74422ae61278202c4e1ad992e8a2e7375227be0a21c6b87ca8d5/pillow-12.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce", size = 6493028, upload-time = "2026-02-11T04:22:42.73Z" },
- { url = "https://files.pythonhosted.org/packages/c6/da/e3c008ed7d2dd1f905b15949325934510b9d1931e5df999bb15972756818/pillow-12.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8", size = 7191940, upload-time = "2026-02-11T04:22:44.543Z" },
- { url = "https://files.pythonhosted.org/packages/01/4a/9202e8d11714c1fc5951f2e1ef362f2d7fbc595e1f6717971d5dd750e969/pillow-12.1.1-cp314-cp314t-win32.whl", hash = "sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36", size = 6438736, upload-time = "2026-02-11T04:22:46.347Z" },
- { url = "https://files.pythonhosted.org/packages/f3/ca/cbce2327eb9885476b3957b2e82eb12c866a8b16ad77392864ad601022ce/pillow-12.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b", size = 7182894, upload-time = "2026-02-11T04:22:48.114Z" },
- { url = "https://files.pythonhosted.org/packages/ec/d2/de599c95ba0a973b94410477f8bf0b6f0b5e67360eb89bcb1ad365258beb/pillow-12.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334", size = 2546446, upload-time = "2026-02-11T04:22:50.342Z" },
- { url = "https://files.pythonhosted.org/packages/56/11/5d43209aa4cb58e0cc80127956ff1796a68b928e6324bbf06ef4db34367b/pillow-12.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:600fd103672b925fe62ed08e0d874ea34d692474df6f4bf7ebe148b30f89f39f", size = 5228606, upload-time = "2026-02-11T04:22:52.106Z" },
- { url = "https://files.pythonhosted.org/packages/5f/d5/3b005b4e4fda6698b371fa6c21b097d4707585d7db99e98d9b0b87ac612a/pillow-12.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:665e1b916b043cef294bc54d47bf02d87e13f769bc4bc5fa225a24b3a6c5aca9", size = 4622321, upload-time = "2026-02-11T04:22:53.827Z" },
- { url = "https://files.pythonhosted.org/packages/df/36/ed3ea2d594356fd8037e5a01f6156c74bc8d92dbb0fa60746cc96cabb6e8/pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:495c302af3aad1ca67420ddd5c7bd480c8867ad173528767d906428057a11f0e", size = 5247579, upload-time = "2026-02-11T04:22:56.094Z" },
- { url = "https://files.pythonhosted.org/packages/54/9a/9cc3e029683cf6d20ae5085da0dafc63148e3252c2f13328e553aaa13cfb/pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8fd420ef0c52c88b5a035a0886f367748c72147b2b8f384c9d12656678dfdfa9", size = 6989094, upload-time = "2026-02-11T04:22:58.288Z" },
- { url = "https://files.pythonhosted.org/packages/00/98/fc53ab36da80b88df0967896b6c4b4cd948a0dc5aa40a754266aa3ae48b3/pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f975aa7ef9684ce7e2c18a3aa8f8e2106ce1e46b94ab713d156b2898811651d3", size = 5313850, upload-time = "2026-02-11T04:23:00.554Z" },
- { url = "https://files.pythonhosted.org/packages/30/02/00fa585abfd9fe9d73e5f6e554dc36cc2b842898cbfc46d70353dae227f8/pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8089c852a56c2966cf18835db62d9b34fef7ba74c726ad943928d494fa7f4735", size = 5963343, upload-time = "2026-02-11T04:23:02.934Z" },
- { url = "https://files.pythonhosted.org/packages/f2/26/c56ce33ca856e358d27fda9676c055395abddb82c35ac0f593877ed4562e/pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e", size = 7029880, upload-time = "2026-02-11T04:23:04.783Z" },
+version = "12.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/25/c2/669d88644cddb1485bd9534e63e8cf476c8e51cb3c3a1297677023505c0e/pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a", size = 5392418, upload-time = "2026-07-01T11:53:27.808Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/ba/3762f376a2948e3036488d773a146e0ae6ecc2ca03ac20e2615bd0b2ba02/pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7", size = 4785287, upload-time = "2026-07-01T11:53:29.761Z" },
+ { url = "https://files.pythonhosted.org/packages/07/50/b5d688cc9c52d4482f3d5bcab6ce20bc2a74a85d2343841c907444a3be2c/pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f", size = 6253754, upload-time = "2026-07-01T11:53:32.298Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/89/36f4cd76cf4baf05c50ababb976249153f18c959171c7f6ba09a6f217260/pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec", size = 6925605, upload-time = "2026-07-01T11:53:34.487Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/c0/4de58cf6633b9e3a6061ef4be6fb91fc3c90b812ece886f531e3c523d777/pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468", size = 6327788, upload-time = "2026-07-01T11:53:36.433Z" },
+ { url = "https://files.pythonhosted.org/packages/87/3c/14d53682a19550dbbaf3b598f807d5457646c510805a44c7d7891cd1cd1a/pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed", size = 7036288, upload-time = "2026-07-01T11:53:38.712Z" },
+ { url = "https://files.pythonhosted.org/packages/38/1d/36279e3c77efe034e4cc2b0393ee74ffdb5a62391dacbf9b916154f5f0b8/pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1", size = 6472396, upload-time = "2026-07-01T11:53:40.781Z" },
+ { url = "https://files.pythonhosted.org/packages/48/7c/8fa0039574c476d7c6fa57dd7c32a130436877c6ec1e5ce1cc8ec44878c1/pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb", size = 7226887, upload-time = "2026-07-01T11:53:42.764Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/17/e324be141d173c1c919428066c3259f21c1b8982e564e01a4a81e96dbdcf/pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f", size = 2568039, upload-time = "2026-07-01T11:53:45.372Z" },
+ { url = "https://files.pythonhosted.org/packages/fb/c8/0a78b0e02d7ac54bc03e5321c9220da52f0c2ea83b21f7c40e7f3169c502/pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756", size = 5392415, upload-time = "2026-07-01T11:53:47.162Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/5b/a02d30018abd97ced9f5a6c63d28597694a00d066516b9c1c6de45859fc9/pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6", size = 4785266, upload-time = "2026-07-01T11:53:49.079Z" },
+ { url = "https://files.pythonhosted.org/packages/c8/98/766667a4be768150a202836acd9fad19c06824ca86c4286d3cf6b274964e/pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd", size = 6263814, upload-time = "2026-07-01T11:53:51.32Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/2d/ede717bc1144f63886c21fd349bb95860b0d1a21149ff16f2bb362b612b6/pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd", size = 6934408, upload-time = "2026-07-01T11:53:53.487Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/48/9c58b685e69d49c31af6c8eb9012055fab7e665785165c84796e2c73ce72/pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c", size = 6337160, upload-time = "2026-07-01T11:53:55.457Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/fa/dc2a5c0ba6df93f67c31d34b808b7ce440b40cdbf96f0b81cde1d1e6fa93/pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5", size = 7045172, upload-time = "2026-07-01T11:53:57.736Z" },
+ { url = "https://files.pythonhosted.org/packages/86/a5/444817a4d4c4c2417df00513086ca196f388d8f9ef40c2e4ccd1ad1af54b/pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b", size = 6472232, upload-time = "2026-07-01T11:53:59.767Z" },
+ { url = "https://files.pythonhosted.org/packages/63/c6/4bad1b18d132a50b27e1365e1ab163616f7a5bb56d330f66f9d1d9d4f9d4/pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a", size = 7233653, upload-time = "2026-07-01T11:54:02.066Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/16/00f91ab7760dc842f5aad55217e80fc4a7067a0604535249bc8a2d6d9870/pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26", size = 2568195, upload-time = "2026-07-01T11:54:04.622Z" },
+ { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" },
+ { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" },
+ { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" },
+ { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" },
+ { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" },
+ { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" },
+ { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" },
+ { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" },
+ { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" },
+ { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" },
+ { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" },
+ { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" },
+ { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" },
+ { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" },
+ { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" },
+ { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" },
+ { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" },
+ { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" },
+ { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" },
+ { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" },
+ { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" },
+ { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" },
+ { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" },
+ { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" },
+ { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" },
+ { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" },
+ { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" },
+ { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" },
+ { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" },
+ { url = "https://files.pythonhosted.org/packages/75/18/2e8b40223153ccbc60df07f9e8928dc0c76202aa4e55ae9f53962b6510d6/pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468", size = 5302510, upload-time = "2026-07-01T11:56:25.736Z" },
+ { url = "https://files.pythonhosted.org/packages/46/3e/51fabf59d5ab801ceab709453d3ab6b180083496579549de4c45ced6528a/pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94", size = 4736058, upload-time = "2026-07-01T11:56:28.041Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/20/22fe9384b7949e25fb1293bcfc84fb82590ff4ea6b37c95b24d26d793d86/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e", size = 5237776, upload-time = "2026-07-01T11:56:30.263Z" },
+ { url = "https://files.pythonhosted.org/packages/08/14/f6ba68107680ffa74b39985f3f30884e41318fbc4250caa423c79b4788bb/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3", size = 5860358, upload-time = "2026-07-01T11:56:32.68Z" },
+ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" },
]
[[package]]
@@ -2573,7 +2583,7 @@ wheels = [
[[package]]
name = "pytest"
-version = "9.0.2"
+version = "9.0.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "colorama", marker = "sys_platform == 'win32'" },
@@ -2584,9 +2594,9 @@ dependencies = [
{ name = "pygments" },
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" },
]
[[package]]