Migrate to datajoint v2

.github/workflows/mysql-integration.yml

@@ -40,7 +40,7 @@ jobs:
           python-version: "3.12"
 
       - name: Install dependencies
-        run: uv pip install --system pytest "setuptools<81" "datajoint==0.14.7" -e .
+        run: uv pip install --system pytest "setuptools<81" -e .
 
       - name: Run MySQL integration tests
         env:

.github/workflows/python-app.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Install dependencies
         run: |
           uv pip install --system flake8 pytest
-          uv pip install --system "setuptools<81" "datajoint==0.14.7" -e .
+          uv pip install --system "setuptools<81" -e .
 
       - name: Lint with flake8
         run: |

README.md

@@ -45,6 +45,21 @@ e.g. based on the template <code>djimaging/djconfig/djconf_template.json</code>.
 Fill out the missing values; if you don't know how, ask someone in your group.
 > ❗ Never upload this personal config file to GitHub.
 
+DataJoint 2 requires MySQL 8 with `utf8mb4`/`utf8mb4_bin` and named stores.
+The template uses a reference-only DataJoint store rooted at
+`/gpfs01/euler/data` for externally managed acquisition, stimulus, and model
+files. DataJoint does not write `<filepath>` values, although filesystem tools
+such as ROI-mask editors may still require write access to adjacent folders.
+Traces, snippets, response vectors, field/presentation channel averages, ROI masks,
+and small analysis arrays/objects use `<blob>` and are stored directly in the
+database. Large stimulus and population matrices, full spatiotemporal receptive
+fields, high-resolution channel averages, correlation maps, and model objects
+use `<npy@processed>` or `<blob@processed>` in the separate writable
+`processed` store. Paths inserted into `<filepath@reference>` fields must be
+relative to the common reference root.
+JSON does not support comments, so these store roles are documented here rather
+than inside the config template.
+
 ### Create a user folder
 
 Inside the root folder <code>djimaging</code> (not in <code>djimaging/djimaging</code>)
@@ -70,10 +85,10 @@ calling <code>schema.drop()</code> and confirm by entering <code>yes</code>.
 
 ## Local testing
 
-Install the test environment with DataJoint 0.14.7:
+Install the test environment with DataJoint 2.3.x:
 
 ```bash
-uv pip install pytest "setuptools<81" "datajoint==0.14.7" -e .
+uv pip install pytest "setuptools<81" -e .
 ```
 
 Run unit tests:
@@ -87,7 +102,8 @@ Run integration tests against a temporary MySQL 8.0.43 server:
 ```bash
 docker run --rm --name djimaging-mysql-test \
   -e MYSQL_ROOT_PASSWORD=datajoint -e MYSQL_ROOT_HOST=% \
-  -p 3307:3306 -d mysql:8.0.43
+  -p 3307:3306 -d mysql:8.0.43 \
+  --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
 until docker exec djimaging-mysql-test mysqladmin ping -h 127.0.0.1 -pdatajoint; do sleep 1; done
 
 DJ_TEST_MYSQL=1 DJ_HOST=127.0.0.1 DJ_PORT=3307 \

djimaging/autorois/autoshift_utils.py

@@ -139,15 +139,18 @@ def compute_corr_map_match_indexes(
     # Iterate over all possible shifts in both x and y directions within the range [-ShiftMax, ShiftMax]
     for i, (dx, dy) in enumerate(dx_dy_pairs, start=1):
         # Crop the shifted_video and stack_corr images to the same size
-        cropped = corr_map[max(0, -dx):min(ref_corr_map.shape[0], ref_corr_map.shape[0] - dx),
-                  max(0, -dy):min(ref_corr_map.shape[1], ref_corr_map.shape[1] - dy)]
+        cropped = corr_map[max(0, -dx):min(corr_map.shape[0], corr_map.shape[0] - dx),
+                  max(0, -dy):min(corr_map.shape[1], corr_map.shape[1] - dy)]
 
         ref_cropped = ref_corr_map[max(0, dx):min(ref_corr_map.shape[0], ref_corr_map.shape[0] + dx),
                       max(0, dy):min(ref_corr_map.shape[1], ref_corr_map.shape[1] + dy)]
 
         # Calculate the correlation coefficient between the cropped images
         if metric == 'corr':
-            score = np.corrcoef(cropped.flatten(), ref_cropped.flatten())[0, 1]
+            a = cropped.ravel() - cropped.mean()
+            b = ref_cropped.ravel() - ref_cropped.mean()
+            denom = np.linalg.norm(a) * np.linalg.norm(b)
+            score = np.dot(a, b) / denom if denom > 0 else 0.0
         elif metric == 'mse':
             score = -np.mean((cropped - ref_cropped) ** 2)
         else:
@@ -165,6 +168,7 @@ def compute_corr_map_match_indexes(
             ax2.imshow(cropped)
 
             plt.show()
+            plt.close(fig)
 
         # Store the correlation coefficient in the cross-correlation matrix
         match_indexes[dx + shift_max, dy + shift_max] = score
@@ -176,9 +180,9 @@ def compute_corr_map_match_indexes(
     v_max = np.max(match_indexes)
     v_max_row_loc, v_max_col_loc = np.unravel_index(np.argmax(match_indexes), match_indexes.shape)
 
-    # Shift the ROI mask
-    x_shift = v_max_row_loc - shift_max
-    y_shift = v_max_col_loc - shift_max
+    # Recover the shift: dx stored at index dx+shift_max, so dx = index-shift_max
+    x_shift = shift_max - v_max_row_loc
+    y_shift = shift_max - v_max_col_loc
 
     if verbose:
         print("ROI mask shifted by", x_shift, "pixels in x and", y_shift, "pixels in y")

djimaging/autorois/cellpose_wrapper.py

@@ -30,7 +30,7 @@ def __init__(self, init_kwargs: dict, eval_kwargs: dict) -> None:
 
         from cellpose import models, io
         io.logger_setup()
-        self.model = models.Cellpose(**self.init_kwargs)
+        self.model = models.CellposeModel(**self.init_kwargs)
 
     @staticmethod
     def stack_to_image(stack: np.ndarray, n_artifact: int) -> np.ndarray:
@@ -139,7 +139,7 @@ def create_mask_from_data(
         if multiple_stacks and 'stitch_threshold' not in eval_kwargs:
             eval_kwargs['stitch_threshold'] = 1e-9  # Stitch everything together
 
-        masks, flows, styles, diams = self.model.eval(imgs, do_3D=do_3D, **eval_kwargs)
+        masks, flows, *_ = self.model.eval(imgs, do_3D=do_3D, **eval_kwargs)
 
         if plot:
             self.plot_results(imgs, masks, flows)