Skip to content

Refactor Scuro tests #2607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shieru1214 wants to merge 5 commits into from
Closed

Refactor Scuro tests #2607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7a0654f
[MINOR] Fix three join defects in Scuro and rewrite the join tests
shieru1214 Sep 3, 2026
bad71a8
[MINOR] Refactor the fusion, text context and scheduler tests
shieru1214 Sep 3, 2026
5d4854c
[MINOR] Merge redundant subtests in the Scuro window operation tests
shieru1214 Sep 3, 2026
e466cb2
[MINOR] Merge per-modality duplicates in the Scuro optimizer tests
shieru1214 Sep 3, 2026
58f663c
[MINOR] Merge the per-loader duplicates in the Scuro data loader tests
shieru1214 Sep 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions src/main/python/systemds/scuro/modality/joined.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def execute(self, starting_idx=0):
self.joined_right.data[i - starting_idx].append([])
right = np.array([])
if self.condition.join_type == "<":
while c < len(idx_2) - 1 and idx_2[c] < nextIdx[j]:
while c < len(idx_2) and idx_2[c] < nextIdx[j]:
if right.size == 0:
right = self.right_modality.data[i][c]
if right.ndim == 1:
Expand All @@ -123,15 +123,21 @@ def execute(self, starting_idx=0):
)
c = c + 1
else:
while c < len(idx_2) - 1 and idx_2[c] <= idx_1[j]:
matches = []
while c < len(idx_2) and idx_2[c] <= idx_1[j]:
if idx_2[c] == idx_1[j]:
right.append(self.right_modality.data[i][c])
match = self.right_modality.data[i][c]
if match.ndim == 1:
match = match[np.newaxis, :]
matches.append(match)
c = c + 1
if matches:
right = np.concatenate(matches, axis=0)

if (
len(right) == 0
): # Audio and video length sometimes do not match so we add the average all audio samples for this specific frame
right = np.mean(self.right_modality.data[i][c - 1 : c], axis=0)
right = np.mean(self.right_modality.data[i], axis=0)
if right.ndim == 1:
right = right[
np.newaxis, :
Expand Down
67 changes: 26 additions & 41 deletions src/main/python/tests/scuro/test_data_loaders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,32 @@ def setUpClass(cls):
def tearDownClass(cls):
shutil.rmtree(cls.test_file_path)

def test_audio_loader_loads_all_instances(self):
loader = AudioLoader(
self.data_generator.get_modality_path(ModalityType.AUDIO),
self.data_generator.indices,
)
data, metadata = loader.load()

self.assertEqual(len(data), self.num_instances)
self.assertEqual(len(metadata), self.num_instances)

for arr in data:
self.assertIsInstance(arr, np.ndarray)
self.assertEqual(arr.ndim, 1)
# Loading is the same contract for every loader -- one array plus one
# metadata entry per instance, at the dimensionality that modality has --
# so the three cases only differed in the loader class and the expected
# ndim. The stats tests below stay separate: each stats class exposes
# different fields, so there is no shared assertion to parameterise.
LOADERS_AND_DIMENSIONS = [
(AudioLoader, ModalityType.AUDIO, 1),
(VideoLoader, ModalityType.VIDEO, 4),
(ImageLoader, ModalityType.IMAGE, 3),
]

def test_loaders_load_all_instances(self):
for loader_class, modality_type, expected_ndim in self.LOADERS_AND_DIMENSIONS:
with self.subTest(loader=loader_class.__name__):
loader = loader_class(
self.data_generator.get_modality_path(modality_type),
self.data_generator.indices,
)
data, metadata = loader.load()

self.assertEqual(len(data), self.num_instances)
self.assertEqual(len(metadata), self.num_instances)

for arr in data:
self.assertIsInstance(arr, np.ndarray)
self.assertEqual(arr.ndim, expected_ndim)

def test_audio_loader_stats(self):
loader = AudioLoader(
Expand All @@ -76,20 +89,6 @@ def test_audio_loader_stats(self):
self.assertEqual(stats.max_length, 44100)
self.assertAlmostEqual(stats.avg_length, (44100 * 2) / 2.0)

def test_video_loader_loads_all_instances(self):
loader = VideoLoader(
self.data_generator.get_modality_path(ModalityType.VIDEO),
self.data_generator.indices,
)
data, metadata = loader.load()

self.assertEqual(len(data), self.num_instances)
self.assertEqual(len(metadata), self.num_instances)

for arr in data:
self.assertIsInstance(arr, np.ndarray)
self.assertEqual(arr.ndim, 4)

def test_video_loader_stats(self):
loader = VideoLoader(
self.data_generator.get_modality_path(ModalityType.VIDEO),
Expand Down Expand Up @@ -129,20 +128,6 @@ def test_text_loader_stats(self):
self.assertEqual(stats.max_length, 7)
self.assertAlmostEqual(stats.avg_length, (7 + 7) / 2.0)

def test_image_loader_loads_all_instances(self):
loader = ImageLoader(
self.data_generator.get_modality_path(ModalityType.IMAGE),
self.data_generator.indices,
)
data, metadata = loader.load()

self.assertEqual(len(data), self.num_instances)
self.assertEqual(len(metadata), self.num_instances)

for arr in data:
self.assertIsInstance(arr, np.ndarray)
self.assertEqual(arr.ndim, 3)

def test_image_loader_stats(self):
loader = ImageLoader(
self.data_generator.get_modality_path(ModalityType.IMAGE),
Expand Down
122 changes: 62 additions & 60 deletions src/main/python/tests/scuro/test_fusion_orders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,77 +19,79 @@
#
# -------------------------------------------------------------

import os
import shutil
import unittest
import numpy as np

from systemds.scuro import Concatenation, RowMax, Hadamard
from systemds.scuro.modality.unimodal_modality import UnimodalModality
from systemds.scuro.representations.bert import Bert
from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
from systemds.scuro.representations.average import Average
from tests.scuro.data_generator import ModalityRandomDataGenerator
from systemds.scuro.modality.type import ModalityType


class TestFusionOrders(unittest.TestCase):
"""
The interesting content is the table below rather than the call sequence:
which operator is commutative, whose result depends on the order of a
pairwise chain, and where a pairwise chain equals the n-ary form. Written as
a table those differences are visible at a glance and a new operator is one
line.
"""

# (operator, chain_order_independent, chain_equals_nary)
# Commutativity is not listed: every Fusion operator declares a
# "commutative" attribute, so the test compares the measured behaviour
# against that declaration instead of against a second copy of it. A new
# operator whose declaration contradicts its implementation fails here
# without anyone having to remember to extend this table.
# Combining a pair is never the same as combining all three, so that case
# is asserted for every operator instead of being listed here.
FUSION_PROPERTIES = [
(Average, True, False),
(Concatenation, False, True),
(RowMax, True, True),
(Hadamard, True, True),
]

@classmethod
def setUpClass(cls):
cls.num_instances = 40
# The properties under test hold for any input shape.
cls.num_instances = 4
cls.num_features = 8
cls.data_generator = ModalityRandomDataGenerator()
cls.r_1 = cls.data_generator.create1DModality(40, 100, ModalityType.AUDIO)
cls.r_2 = cls.data_generator.create1DModality(40, 100, ModalityType.TEXT)
cls.r_3 = cls.data_generator.create1DModality(40, 100, ModalityType.TEXT)

def test_fusion_order_avg(self):
r_1_r_2 = self.r_1.combine(self.r_2, Average())
r_2_r_1 = self.r_2.combine(self.r_1, Average())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Average())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Average())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Average())

self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertFalse(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))

def test_fusion_order_concat(self):
r_1_r_2 = self.r_1.combine(self.r_2, Concatenation())
r_2_r_1 = self.r_2.combine(self.r_1, Concatenation())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Concatenation())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Concatenation())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Concatenation())

self.assertFalse(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertFalse(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertFalse(np.array_equal(r_2_r_1.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))

def test_fusion_order_max(self):
r_1_r_2 = self.r_1.combine(self.r_2, RowMax())
r_2_r_1 = self.r_2.combine(self.r_1, RowMax())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, RowMax())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, RowMax())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], RowMax())

self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))

def test_fusion_order_hadamard(self):
r_1_r_2 = self.r_1.combine(self.r_2, Hadamard())
r_2_r_1 = self.r_2.combine(self.r_1, Hadamard())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Hadamard())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Hadamard())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Hadamard())

self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))
def setUp(self):
self.r_1 = self.data_generator.create1DModality(
self.num_instances, self.num_features, ModalityType.AUDIO
)
self.r_2 = self.data_generator.create1DModality(
self.num_instances, self.num_features, ModalityType.TEXT
)
self.r_3 = self.data_generator.create1DModality(
self.num_instances, self.num_features, ModalityType.TEXT
)

@staticmethod
def _equal(left, right):
return np.array_equal(np.asarray(left.data), np.asarray(right.data))

def test_fusion_order_properties(self):
for (
fusion_operator,
chain_order_independent,
chain_equals_nary,
) in self.FUSION_PROPERTIES:
with self.subTest(fusion=fusion_operator.__name__):
r_1_r_2 = self.r_1.combine(self.r_2, fusion_operator())
r_2_r_1 = self.r_2.combine(self.r_1, fusion_operator())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, fusion_operator())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, fusion_operator())
r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], fusion_operator())

self.assertEqual(
self._equal(r_1_r_2, r_2_r_1), fusion_operator().commutative
)
self.assertEqual(
self._equal(r_1_r_2_r_3, r_2_r_1_r_3), chain_order_independent
)
self.assertEqual(self._equal(r_1_r_2_r_3, r1_r2_r3), chain_equals_nary)
self.assertFalse(self._equal(r_1_r_2, r1_r2_r3))
42 changes: 23 additions & 19 deletions src/main/python/tests/scuro/test_hp_tuner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,16 +74,31 @@ def setUpClass(cls):
TestTask("UnimodalRepresentationTask2", "TestSVM2", cls.num_instances),
]

def test_hp_tuner_for_text_modality(self):
text_data, text_md = ModalityRandomDataGenerator().create_text_data(
self.num_instances
)
text = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.TEXT, text_data, str, text_md
def _create_modality(self, modality_type):
if modality_type is ModalityType.TEXT:
data, metadata = ModalityRandomDataGenerator().create_text_data(
self.num_instances
)
data_type = str
else:
data, metadata = ModalityRandomDataGenerator().create_visual_modality(
self.num_instances, 1
)
data_type = np.float32

return UnimodalModality(
TestDataLoader(self.indices, None, modality_type, data, data_type, metadata)
)
self.run_hp_for_modality([text])

def test_hp_tuner_per_modality(self):
# The text and image cases ran the same tuner over the same registry and
# asserted the same thing -- every assertion lives in
# run_hp_for_modality, so the two tests differed only in how the
# modality is built. Building it is a factory and the modality type is a
# subTest dimension; both cases still run and are reported separately.
for modality_type in [ModalityType.TEXT, ModalityType.IMAGE]:
with self.subTest(modality=modality_type.name):
self.run_hp_for_modality([self._create_modality(modality_type)])

# TODO: Add once the final multimodal optimizer is implemented
# def test_multimodal_hp_tuning(self):
Expand All @@ -109,17 +124,6 @@ def test_hp_tuner_for_text_modality(self):
# [audio, text], multimodal=True, tune_unimodal_representations=False
# )

def test_hp_tuner_for_image_modality(self):
image_data, image_md = ModalityRandomDataGenerator().create_visual_modality(
self.num_instances, 1
)
image = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.IMAGE, image_data, np.float32, image_md
)
)
self.run_hp_for_modality([image])

def run_hp_for_modality(
self, modalities, multimodal=False, tune_unimodal_representations=False
):
Expand Down
Loading
Loading