From 0d2f2cfb89e592edf94798e79a47213ae457aadd Mon Sep 17 00:00:00 2001 From: Michal Torma Date: Thu, 15 Oct 2026 19:58:22 +0200 Subject: [PATCH 1/3] Fix(AppResources): merge UIFormatters with backstop defaults Custom collection/discipline UIFormatters no longer fully replace system catalog-number formats. Missing formats are filled from backstop by name (Fixes #8676), matching the DataObjFormatters fall-back pattern. Co-authored-by: Cursor --- specifyweb/backend/context/app_resource.py | 74 ++++++++++++----- .../context/tests/test_app_resource.py | 81 ++++++++++++++++++- 2 files changed, 136 insertions(+), 19 deletions(-) diff --git a/specifyweb/backend/context/app_resource.py b/specifyweb/backend/context/app_resource.py index 53be974152a..2d7818af793 100644 --- a/specifyweb/backend/context/app_resource.py +++ b/specifyweb/backend/context/app_resource.py @@ -68,12 +68,14 @@ def get_app_resource(collection, user, resource_name, additional_default=False): if resource_name == 'DataViewQueries': return get_data_view_queries_resource(collection, user) - # Handling for DataObjFormatters to support fallback to defaults - if resource_name == 'DataObjFormatters': + # Handling for DataObjFormatters / UIFormatters to support fallback to defaults. + # Custom resources at a more-specific level would otherwise fully replace + # backstop definitions (see #6243, #8676). + if resource_name in ('DataObjFormatters', 'UIFormatters'): custom_formatter = None default_formatter = None - - # Try to search for the the custom formatter first + + # Try to search for the custom formatter first for level in DIR_LEVELS: # First look in the database. from_db = get_app_resource_from_db(collection, user, level, resource_name) @@ -86,24 +88,33 @@ def get_app_resource(collection, user, resource_name, additional_default=False): if from_fs is not None: custom_formatter = from_fs break - + # If custom formatter not found, try to load the default formatter from the backstop level default_formatter = load_resource_at_level(collection, user, 'Backstop', resource_name) - + # If both custom and default formatters are found, merge them if custom_formatter is not None and default_formatter is not None: - merged_formatter = merge_formatters(custom_formatter[0], default_formatter[0]) + if resource_name == 'UIFormatters': + # Merge by format name: many UIFormatters share the same class + # (e.g. several CatalogNumber* formats on CollectionObject). + merged_formatter = merge_ui_formatters( + custom_formatter[0], default_formatter[0] + ) + else: + merged_formatter = merge_formatters( + custom_formatter[0], default_formatter[0] + ) return merged_formatter, custom_formatter[1], custom_formatter[2] - + # Return the custom formatter if it exists, otherwise return the default if custom_formatter is not None: return custom_formatter - + if default_formatter is not None: return default_formatter - + return None - + if not additional_default: # Traverse the hierarchy. for level in DIR_LEVELS: @@ -323,15 +334,15 @@ def get_app_resource_dirs_for_level(collection, user, level): return Spappresourcedir.objects.filter(**filters) def merge_formatters(custom_xml, default_xml): - """Merge custom and default XML formatters. - This function takes two XML strings, one for custom formatters and one for default formatters, - and merges them by adding any formatters from the default XML that are not already present in - the custom XML. It returns the merged XML as a string. + """Merge custom and default XML DataObjFormatters by ``class``. + + Adds any formatters from the default XML whose ``class`` is not already + present in the custom XML. Returns the merged XML as a string. """ - + custom_root = ElementTree.fromstring(custom_xml) default_root = ElementTree.fromstring(default_xml) - + # Create a set of existing classes in the custom XML to avoid duplicates custom_classes = set() for format_elem in custom_root.findall('format'): @@ -344,6 +355,33 @@ def merge_formatters(custom_xml, default_xml): class_attr = format_elem.get('class') if class_attr and class_attr not in custom_classes: custom_root.append(format_elem) - + # Convert the merged XML tree back to a string return ElementTree.tostring(custom_root, encoding='utf-8').decode('utf-8') + + +def merge_ui_formatters(custom_xml, default_xml): + """Merge custom and default UIFormatters by format ``name``. + + Unlike DataObjFormatters (merged by ``class``), several UIFormatters can + share the same datamodel class (e.g. multiple catalog-number formats on + CollectionObject). Custom entries with the same name win; missing names + are filled from the default/backstop resource. + """ + + custom_root = ElementTree.fromstring(custom_xml) + default_root = ElementTree.fromstring(default_xml) + + custom_names = { + format_elem.get('name') + for format_elem in custom_root.findall('format') + if format_elem.get('name') + } + + for format_elem in default_root.findall('format'): + name = format_elem.get('name') + if name and name not in custom_names: + custom_root.append(format_elem) + custom_names.add(name) + + return ElementTree.tostring(custom_root, encoding='utf-8').decode('utf-8') diff --git a/specifyweb/backend/context/tests/test_app_resource.py b/specifyweb/backend/context/tests/test_app_resource.py index 900576db564..c1f31159f7f 100644 --- a/specifyweb/backend/context/tests/test_app_resource.py +++ b/specifyweb/backend/context/tests/test_app_resource.py @@ -1,13 +1,44 @@ +from xml.etree import ElementTree + from django.test import Client from specifyweb.specify.tests.test_api import ApiTests from unittest.mock import Mock, patch -from specifyweb.backend.context.app_resource import get_data_view_queries_resource +from specifyweb.backend.context.app_resource import ( + get_app_resource, + get_data_view_queries_resource, + merge_ui_formatters, +) FOUND_RESOURCE = ('"Value"', "text/xml", 4) +CUSTOM_UI_FORMATTERS = """ + + + + + + + + +""" + +BACKSTOP_UI_FORMATTERS = """ + + + + + + edu.ku.brc.specify.ui.CatalogNumberUIFieldFormatter + + + edu.ku.brc.specify.ui.CatalogNumberStringUIFieldFormatter + + +""" + class TestAppResource(ApiTests): @@ -90,3 +121,51 @@ def test_not_found_quiet(self, get_app_resource: Mock): response = c.get("/context/app.resource?name='simple'&quiet='true'") self._assertStatusCodeEqual(response, 204) + + def test_merge_ui_formatters_by_name(self): + merged = merge_ui_formatters(CUSTOM_UI_FORMATTERS, BACKSTOP_UI_FORMATTERS) + root = ElementTree.fromstring(merged) + names = [fmt.get('name') for fmt in root.findall('format')] + + self.assertEqual( + names, + ['GridRefMGRS', 'CatalogNumber', 'CatalogNumberNumeric', 'CatalogNumberString'], + ) + # Custom CatalogNumber wins over backstop (keeps numeric part, not year). + catalog_number = root.find("./format[@name='CatalogNumber']") + self.assertIsNotNone(catalog_number) + self.assertIsNotNone(catalog_number.find("./field[@type='numeric']")) + self.assertIsNone(catalog_number.find("./field[@type='year']")) + + @patch("specifyweb.backend.context.app_resource.load_resource_at_level") + @patch("specifyweb.backend.context.app_resource.get_app_resource_from_db") + def test_uiformatters_merge_custom_with_backstop( + self, get_from_db: Mock, load_from_filesystem: Mock + ): + get_from_db.side_effect = lambda _collection, _user, level, _name: ( + (CUSTOM_UI_FORMATTERS, 'text/xml', 23) if level == 'Collection' else None + ) + load_from_filesystem.side_effect = lambda _collection, _user, level, _name: ( + (BACKSTOP_UI_FORMATTERS, 'text/xml', None) if level == 'Backstop' else None + ) + + result = get_app_resource(self.collection, self.specifyuser, 'UIFormatters') + + self.assertIsNotNone(result) + data, mimetype, resource_id = result + self.assertEqual(mimetype, 'text/xml') + self.assertEqual(resource_id, 23) + + names = { + fmt.get('name') + for fmt in ElementTree.fromstring(data).findall('format') + } + self.assertEqual( + names, + { + 'GridRefMGRS', + 'CatalogNumber', + 'CatalogNumberNumeric', + 'CatalogNumberString', + }, + ) From 0cd925425b578f19b02171581a9c5be279c90ea5 Mon Sep 17 00:00:00 2001 From: Michal Torma Date: Fri, 16 Oct 2026 12:08:31 +0200 Subject: [PATCH 2/3] Enhance(UIFormatters): support additional default loading and merging logic Updated the get_app_resource function to handle UIFormatters with an additional_default flag, allowing for the retrieval of Backstop formats without merging custom resources. Added tests to verify the new behavior, ensuring that formats are correctly merged and that custom formats do not interfere when the additional_default flag is set. Co-authored-by: Cursor --- specifyweb/backend/context/app_resource.py | 58 +++++++++--- .../context/tests/test_app_resource.py | 94 +++++++++++++++++++ 2 files changed, 140 insertions(+), 12 deletions(-) diff --git a/specifyweb/backend/context/app_resource.py b/specifyweb/backend/context/app_resource.py index 2d7818af793..947d0da802c 100644 --- a/specifyweb/backend/context/app_resource.py +++ b/specifyweb/backend/context/app_resource.py @@ -68,12 +68,17 @@ def get_app_resource(collection, user, resource_name, additional_default=False): if resource_name == 'DataViewQueries': return get_data_view_queries_resource(collection, user) + # UIFormatters + additional_default: return Backstop only (same contract as + # TypeSearches with additionaldefault=true). Do not merge custom resources. + if resource_name == 'UIFormatters' and additional_default: + return load_resource_at_level(collection, user, 'Backstop', resource_name) + # Handling for DataObjFormatters / UIFormatters to support fallback to defaults. # Custom resources at a more-specific level would otherwise fully replace # backstop definitions (see #6243, #8676). if resource_name in ('DataObjFormatters', 'UIFormatters'): custom_formatter = None - default_formatter = None + custom_level = None # Try to search for the custom formatter first for level in DIR_LEVELS: @@ -81,29 +86,58 @@ def get_app_resource(collection, user, resource_name, additional_default=False): from_db = get_app_resource_from_db(collection, user, level, resource_name) if from_db is not None: custom_formatter = from_db + custom_level = level break # If resource was not found, look on the filesystem. from_fs = load_resource_at_level(collection, user, level, resource_name) if from_fs is not None: custom_formatter = from_fs + custom_level = level break # If custom formatter not found, try to load the default formatter from the backstop level default_formatter = load_resource_at_level(collection, user, 'Backstop', resource_name) - # If both custom and default formatters are found, merge them + if resource_name == 'UIFormatters': + # Merge by format name: many UIFormatters share the same class + # (e.g. several CatalogNumber* formats on CollectionObject). + # Order: most-specific custom → Common → Backstop so overlapping + # names keep the most-specific definition (#8676). + if custom_formatter is not None: + merged_xml = custom_formatter[0] + + # When custom is above Common, also pull Common definitions. + if custom_level not in ('Common', 'Backstop'): + common_formatter = get_app_resource_from_db( + collection, user, 'Common', resource_name + ) + if common_formatter is None: + common_formatter = load_resource_at_level( + collection, user, 'Common', resource_name + ) + if common_formatter is not None: + merged_xml = merge_ui_formatters( + merged_xml, common_formatter[0] + ) + + if default_formatter is not None: + merged_xml = merge_ui_formatters( + merged_xml, default_formatter[0] + ) + + return merged_xml, custom_formatter[1], custom_formatter[2] + + if default_formatter is not None: + return default_formatter + + return None + + # DataObjFormatters: merge by class with Backstop only. if custom_formatter is not None and default_formatter is not None: - if resource_name == 'UIFormatters': - # Merge by format name: many UIFormatters share the same class - # (e.g. several CatalogNumber* formats on CollectionObject). - merged_formatter = merge_ui_formatters( - custom_formatter[0], default_formatter[0] - ) - else: - merged_formatter = merge_formatters( - custom_formatter[0], default_formatter[0] - ) + merged_formatter = merge_formatters( + custom_formatter[0], default_formatter[0] + ) return merged_formatter, custom_formatter[1], custom_formatter[2] # Return the custom formatter if it exists, otherwise return the default diff --git a/specifyweb/backend/context/tests/test_app_resource.py b/specifyweb/backend/context/tests/test_app_resource.py index c1f31159f7f..ab9fb85dab0 100644 --- a/specifyweb/backend/context/tests/test_app_resource.py +++ b/specifyweb/backend/context/tests/test_app_resource.py @@ -25,6 +25,21 @@ """ +COMMON_UI_FORMATTERS = """ + + + + + + + + + + + + +""" + BACKSTOP_UI_FORMATTERS = """ @@ -169,3 +184,82 @@ def test_uiformatters_merge_custom_with_backstop( 'CatalogNumberString', }, ) + + @patch("specifyweb.backend.context.app_resource.load_resource_at_level") + @patch("specifyweb.backend.context.app_resource.get_app_resource_from_db") + def test_uiformatters_merge_common_before_backstop( + self, get_from_db: Mock, load_from_filesystem: Mock + ): + get_from_db.side_effect = lambda _collection, _user, level, _name: { + 'Collection': (CUSTOM_UI_FORMATTERS, 'text/xml', 23), + 'Common': (COMMON_UI_FORMATTERS, 'text/xml', 11), + }.get(level) + load_from_filesystem.side_effect = lambda _collection, _user, level, _name: ( + (BACKSTOP_UI_FORMATTERS, 'text/xml', None) if level == 'Backstop' else None + ) + + result = get_app_resource(self.collection, self.specifyuser, 'UIFormatters') + + self.assertIsNotNone(result) + data, mimetype, resource_id = result + self.assertEqual(mimetype, 'text/xml') + self.assertEqual(resource_id, 23) + + root = ElementTree.fromstring(data) + names = [fmt.get('name') for fmt in root.findall('format')] + self.assertEqual( + names, + [ + 'GridRefMGRS', + 'CatalogNumber', + 'AccessionNumber', + 'CatalogNumberNumeric', + 'CatalogNumberString', + ], + ) + # Collection CatalogNumber wins over Common and Backstop. + catalog_number = root.find("./format[@name='CatalogNumber']") + self.assertIsNotNone(catalog_number.find("./field[@type='numeric']")) + self.assertIsNone(catalog_number.find("./field[@type='regex']")) + self.assertIsNone(catalog_number.find("./field[@type='year']")) + + @patch("specifyweb.backend.context.app_resource.load_resource_at_level") + @patch("specifyweb.backend.context.app_resource.get_app_resource_from_db") + def test_uiformatters_additional_default_skips_custom( + self, get_from_db: Mock, load_from_filesystem: Mock + ): + get_from_db.return_value = (CUSTOM_UI_FORMATTERS, 'text/xml', 23) + load_from_filesystem.return_value = ( + BACKSTOP_UI_FORMATTERS, + 'text/xml', + None, + ) + + result = get_app_resource( + self.collection, + self.specifyuser, + 'UIFormatters', + additional_default=True, + ) + + self.assertIsNotNone(result) + data, mimetype, resource_id = result + self.assertEqual(mimetype, 'text/xml') + self.assertIsNone(resource_id) + names = { + fmt.get('name') + for fmt in ElementTree.fromstring(data).findall('format') + } + self.assertEqual( + names, + { + 'CatalogNumber', + 'CatalogNumberNumeric', + 'CatalogNumberString', + }, + ) + self.assertNotIn('GridRefMGRS', names) + get_from_db.assert_not_called() + load_from_filesystem.assert_called_once_with( + self.collection, self.specifyuser, 'Backstop', 'UIFormatters' + ) From 2b9dd485212721e97f97cb96200fb557d7c20c5f Mon Sep 17 00:00:00 2001 From: Michal Torma Date: Fri, 16 Oct 2026 12:15:15 +0200 Subject: [PATCH 3/3] Enhance(UIFormatters): add AccessionNumber format and update tests Introduced a new UIFormatter for AccessionNumber, allowing it to be recognized and processed alongside existing formats. Updated unit tests to ensure the new format is correctly integrated and validated against expected behaviors, including checks for field types and merging logic with existing formats. --- .../backend/context/tests/test_app_resource.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/specifyweb/backend/context/tests/test_app_resource.py b/specifyweb/backend/context/tests/test_app_resource.py index ab9fb85dab0..b506f710978 100644 --- a/specifyweb/backend/context/tests/test_app_resource.py +++ b/specifyweb/backend/context/tests/test_app_resource.py @@ -45,6 +45,9 @@ + + + edu.ku.brc.specify.ui.CatalogNumberUIFieldFormatter @@ -144,7 +147,13 @@ def test_merge_ui_formatters_by_name(self): self.assertEqual( names, - ['GridRefMGRS', 'CatalogNumber', 'CatalogNumberNumeric', 'CatalogNumberString'], + [ + 'GridRefMGRS', + 'CatalogNumber', + 'AccessionNumber', + 'CatalogNumberNumeric', + 'CatalogNumberString', + ], ) # Custom CatalogNumber wins over backstop (keeps numeric part, not year). catalog_number = root.find("./format[@name='CatalogNumber']") @@ -180,6 +189,7 @@ def test_uiformatters_merge_custom_with_backstop( { 'GridRefMGRS', 'CatalogNumber', + 'AccessionNumber', 'CatalogNumberNumeric', 'CatalogNumberString', }, @@ -222,6 +232,10 @@ def test_uiformatters_merge_common_before_backstop( self.assertIsNotNone(catalog_number.find("./field[@type='numeric']")) self.assertIsNone(catalog_number.find("./field[@type='regex']")) self.assertIsNone(catalog_number.find("./field[@type='year']")) + # AccessionNumber is only in Common/Backstop: Common wins over Backstop. + accession_number = root.find("./format[@name='AccessionNumber']") + self.assertIsNotNone(accession_number.find("./field[@type='year']")) + self.assertIsNone(accession_number.find("./field[@type='regex']")) @patch("specifyweb.backend.context.app_resource.load_resource_at_level") @patch("specifyweb.backend.context.app_resource.get_app_resource_from_db") @@ -254,6 +268,7 @@ def test_uiformatters_additional_default_skips_custom( names, { 'CatalogNumber', + 'AccessionNumber', 'CatalogNumberNumeric', 'CatalogNumberString', },