diff --git a/layers/fixture_import.py b/layers/fixture_import.py new file mode 100644 index 0000000..cc47562 --- /dev/null +++ b/layers/fixture_import.py @@ -0,0 +1,247 @@ +"""Fixture import scaffolding for phased TDD implementation. + +This module intentionally exposes stable API entrypoints before behavior is +implemented. The eventual importer is root-agnostic and treats incoming rows as +one graph, whether exported from one layer or multiple selected layers. +""" + +from __future__ import annotations + +from django.apps import apps +from django.contrib.sites.models import Site +from django.db import transaction + +from .fixture_contract import ( + NODE_FIELDS_KEY, + NODE_MODEL_KEY, + NODE_RELATIONS_KEY, + NODE_UUID_KEY, + normalize_uuid, +) + + +LAYER_MODEL = "layers.layer" +MULTILAYER_ASSOCIATION_MODEL = "layers.multilayerassociation" + + +def _model_manager(model_class): + """Return an unscoped manager for import-time identity resolution.""" + if hasattr(model_class, "all_objects"): + return model_class.all_objects + return model_class._base_manager + + +def _assert_strict_policies(missing_ref_policy, duplicate_uuid_policy): + if missing_ref_policy != "error": + raise ValueError("Unsupported missing_ref_policy: %s" % missing_ref_policy) + if duplicate_uuid_policy != "error": + raise ValueError("Unsupported duplicate_uuid_policy: %s" % duplicate_uuid_policy) + + +def _row_identity_key(row): + model_label = row.get(NODE_MODEL_KEY) + uuid_value = normalize_uuid(row.get(NODE_UUID_KEY)) + return (model_label, uuid_value) + + +def _assert_no_duplicate_uuid_conflicts(rows): + seen = {} + for row in rows: + model_label, uuid_value = _row_identity_key(row) + if uuid_value is None: + continue + + key = (model_label, uuid_value) + signature = { + NODE_FIELDS_KEY: row.get(NODE_FIELDS_KEY, {}), + NODE_RELATIONS_KEY: row.get(NODE_RELATIONS_KEY, {}), + } + previous = seen.get(key) + if previous is None: + seen[key] = signature + continue + + if previous != signature: + raise ValueError( + "Conflicting duplicate UUID rows for %s (%s)" % (model_label, uuid_value) + ) + + +def _apply_fields(instance, fields): + for field_name, field_value in fields.items(): + if hasattr(instance, field_name): + setattr(instance, field_name, field_value) + + +def _resolve_ref_instance(ref_obj, missing_ref_policy): + ref_model_label = ref_obj.get(NODE_MODEL_KEY) + ref_uuid_value = normalize_uuid(ref_obj.get(NODE_UUID_KEY)) + if not ref_uuid_value: + if missing_ref_policy == "error": + raise ValueError("Missing UUID in relation ref for model %s" % ref_model_label) + return None + + model_class = apps.get_model(ref_model_label) + manager = _model_manager(model_class) + try: + return manager.get(uuid=ref_uuid_value) + except model_class.DoesNotExist: + if missing_ref_policy == "error": + raise ValueError( + "Missing related object for %s UUID %s" % (ref_model_label, ref_uuid_value) + ) + return None + + +def import_fixture_rows( + rows, + dry_run=False, + associate_all_sites=True, + missing_ref_policy="error", + duplicate_uuid_policy="error", +): + """Import fixture rows using UUID-first identity semantics. + + The engine is root-agnostic and processes a single graph composed from all + rows in the fixture. + + Policy defaults are strict to keep behavior deterministic: + - missing_ref_policy: "error" (unsupported: any other value) + - duplicate_uuid_policy: "error" (unsupported: any other value) + + Planned coverage with shared graph engine: + - PR05/PR06: layer import behavior + - PR07: multilayer import behavior + - PR09: theme import behavior + """ + _assert_strict_policies(missing_ref_policy, duplicate_uuid_policy) + rows = rows or [] + _assert_no_duplicate_uuid_conflicts(rows) + + Layer = apps.get_model(LAYER_MODEL) + MultilayerAssociation = apps.get_model(MULTILAYER_ASSOCIATION_MODEL) + layer_manager = _model_manager(Layer) + association_manager = _model_manager(MultilayerAssociation) + + def _execute_import(): + # First pass: upsert layer rows by UUID. + for row in rows: + if row.get(NODE_MODEL_KEY) != LAYER_MODEL: + continue + + layer_uuid = normalize_uuid(row.get(NODE_UUID_KEY)) + if not layer_uuid: + raise ValueError("Layer row missing UUID") + + layer_fields = dict(row.get(NODE_FIELDS_KEY, {})) + layer_obj = layer_manager.filter(uuid=layer_uuid).first() + is_new = layer_obj is None + if is_new: + layer_obj = Layer(uuid=layer_uuid) + + _apply_fields(layer_obj, layer_fields) + layer_obj.save() + + if associate_all_sites: + layer_obj.site.set(Site.objects.all()) + + # Second pass: upsert multilayer associations and resolve FKs by UUID refs. + for row in rows: + if row.get(NODE_MODEL_KEY) != MULTILAYER_ASSOCIATION_MODEL: + continue + + assoc_uuid = normalize_uuid(row.get(NODE_UUID_KEY)) + if not assoc_uuid: + raise ValueError("MultilayerAssociation row missing UUID") + + relations = row.get(NODE_RELATIONS_KEY, {}) + parent_ref = relations.get("parentLayer") + layer_ref = relations.get("layer") + + if not parent_ref: + raise ValueError("Missing parentLayer relation for MultilayerAssociation") + + parent_layer_obj = _resolve_ref_instance(parent_ref, missing_ref_policy) + layer_obj = None + if layer_ref is not None: + layer_obj = _resolve_ref_instance(layer_ref, missing_ref_policy) + + assoc_obj = association_manager.filter(uuid=assoc_uuid).first() + if assoc_obj is None: + assoc_obj = MultilayerAssociation(uuid=assoc_uuid) + + _apply_fields(assoc_obj, row.get(NODE_FIELDS_KEY, {})) + assoc_obj.parentLayer = parent_layer_obj + assoc_obj.layer = layer_obj + assoc_obj.save() + + if dry_run: + with transaction.atomic(): + _execute_import() + transaction.set_rollback(True) + return { + "imported": 0, + "dry_run": True, + } + + _execute_import() + return { + "imported": len(rows), + "dry_run": False, + } + + +def import_layer_rows( + rows, + dry_run=False, + associate_all_sites=True, + missing_ref_policy="error", + duplicate_uuid_policy="error", +): + """Import only layers.layer fixture rows (UUID-first resolution).""" + layer_rows = [row for row in (rows or []) if row.get(NODE_MODEL_KEY) == LAYER_MODEL] + return import_fixture_rows( + layer_rows, + dry_run=dry_run, + associate_all_sites=associate_all_sites, + missing_ref_policy=missing_ref_policy, + duplicate_uuid_policy=duplicate_uuid_policy, + ) + + +def import_multilayer_rows( + rows, + dry_run=False, + missing_ref_policy="error", + duplicate_uuid_policy="error", +): + """Import multilayer-related rows (dimensions, values, associations).""" + multilayer_rows = [ + row + for row in (rows or []) + if row.get(NODE_MODEL_KEY) in {LAYER_MODEL, MULTILAYER_ASSOCIATION_MODEL} + ] + return import_fixture_rows( + multilayer_rows, + dry_run=dry_run, + associate_all_sites=True, + missing_ref_policy=missing_ref_policy, + duplicate_uuid_policy=duplicate_uuid_policy, + ) + + +def import_theme_rows( + rows, + dry_run=False, + associate_all_sites=True, + missing_ref_policy="error", + duplicate_uuid_policy="error", +): + """Import theme-related rows and relations.""" + return import_fixture_rows( + rows, + dry_run=dry_run, + associate_all_sites=associate_all_sites, + missing_ref_policy=missing_ref_policy, + duplicate_uuid_policy=duplicate_uuid_policy, + ) diff --git a/layers/tests/test_fixture_import.py b/layers/tests/test_fixture_import.py new file mode 100644 index 0000000..449a85a --- /dev/null +++ b/layers/tests/test_fixture_import.py @@ -0,0 +1,227 @@ +from uuid import uuid4 + +from django.contrib.sites.models import Site +from django.test import TestCase + +from layers.fixture_contract import build_node, build_ref +from layers.models import Layer, MultilayerAssociation + +try: + from layers.fixture_import import import_fixture_rows +except ImportError: + import_fixture_rows = None + + +class LayerFixtureImportPR05Test(TestCase): + """layer fixture contract tests for UUID-first fixture import behavior.""" + + def _require_importer(self): + self.assertIsNotNone( + import_fixture_rows, + "importer API missing: expected layers.fixture_import.import_fixture_rows", + ) + + def _layer_fields(self, name): + return { + "name": name, + "layer_type": "WMS", + "slug_name": None, + "url": None, + } + + def _import_kwargs(self): + return { + "dry_run": False, + "associate_all_sites": True, + "missing_ref_policy": "error", + "duplicate_uuid_policy": "error", + } + + def test_uuid_match_updates_existing_even_when_source_pk_differs(self): + """Ensure UUIDs are used as the true source of identity, not source PKs.""" + self._require_importer() + + layer_uuid = uuid4() + existing_layer = Layer.objects.create( + name="Original", + layer_type="WMS", + uuid=layer_uuid, + ) + + fixture_rows = [ + build_node( + model="layers.layer", + source_pk=9999, + uuid_value=layer_uuid, + fields=self._layer_fields("Updated by UUID"), + relations={}, + ) + ] + + import_fixture_rows(fixture_rows, **self._import_kwargs()) + + existing_layer.refresh_from_db() + self.assertEqual(existing_layer.name, "Updated by UUID") + self.assertEqual(Layer.objects.filter(uuid=layer_uuid).count(), 1) + + def test_source_pk_collision_with_different_uuid_creates_new_record(self): + """If records with different IDs, but same UUID/type are found, create + a new record with a new ID.""" + self._require_importer() + + existing_layer = Layer.objects.create(name="Existing", layer_type="WMS") + new_uuid = uuid4() + + fixture_rows = [ + build_node( + model="layers.layer", + source_pk=existing_layer.pk, + uuid_value=new_uuid, + fields=self._layer_fields("Created on UUID mismatch"), + relations={}, + ) + ] + + before_count = Layer.objects.count() + import_fixture_rows(fixture_rows, **self._import_kwargs()) + + self.assertEqual(Layer.objects.count(), before_count + 1) + self.assertTrue(Layer.objects.filter(uuid=new_uuid).exists()) + existing_layer.refresh_from_db() + self.assertEqual(existing_layer.name, "Existing") + + def test_second_pass_resolves_relations_by_uuid_not_source_pk(self): + """As name suggests - ensure 2nd pass uses UUIDs for reference, not just PK or 'id'.""" + self._require_importer() + + parent_uuid = uuid4() + target_uuid = uuid4() + association_uuid = uuid4() + + fixture_rows = [ + build_node( + model="layers.layer", + source_pk=101, + uuid_value=parent_uuid, + fields=self._layer_fields("Imported Parent"), + relations={}, + ), + build_node( + model="layers.layer", + source_pk=202, + uuid_value=target_uuid, + fields=self._layer_fields("Imported Target"), + relations={}, + ), + build_node( + model="layers.multilayerassociation", + source_pk=303, + uuid_value=association_uuid, + fields={"name": "Val-1aVal-2b"}, + relations={ + "parentLayer": build_ref( + model="layers.layer", + source_pk=99901, + uuid_value=parent_uuid, + ), + "layer": build_ref( + model="layers.layer", + source_pk=99902, + uuid_value=target_uuid, + ), + }, + ), + ] + + import_fixture_rows(fixture_rows, **self._import_kwargs()) + + imported_parent = Layer.objects.get(uuid=parent_uuid) + imported_target = Layer.objects.get(uuid=target_uuid) + imported_association = MultilayerAssociation.objects.get(uuid=association_uuid) + + self.assertEqual(imported_association.parentLayer_id, imported_parent.pk) + self.assertEqual(imported_association.layer_id, imported_target.pk) + + def test_new_layers_are_associated_to_all_sites_by_default(self): + """Sites info should no longer matter on import if all DBs are segregated. + We will assume any imported layer is intended to be seen on the new server, + so we will associate it with all sites by default.""" + self._require_importer() + + Site.objects.get_or_create(id=1, defaults={"domain": "example.com", "name": "example"}) + Site.objects.get_or_create(id=2, defaults={"domain": "preview.example.com", "name": "preview"}) + + new_uuid = uuid4() + fixture_rows = [ + build_node( + model="layers.layer", + source_pk=404, + uuid_value=new_uuid, + fields=self._layer_fields("Site-linked Import"), + relations={}, + ) + ] + + import_fixture_rows(fixture_rows, **self._import_kwargs()) + + imported_layer = Layer.objects.get(uuid=new_uuid) + imported_site_ids = set(imported_layer.site.values_list("id", flat=True)) + all_site_ids = set(Site.objects.values_list("id", flat=True)) + self.assertEqual(imported_site_ids, all_site_ids) + + def test_duplicate_uuid_rows_with_conflicting_fields_raise_error(self): + """two records with the same UUID and a different field. Right now + we don't have a plan for resolving this, so ValueError should be raised.""" + self._require_importer() + + shared_uuid = uuid4() + fixture_rows = [ + build_node( + model="layers.layer", + source_pk=501, + uuid_value=shared_uuid, + fields=self._layer_fields("Name A"), + relations={}, + ), + build_node( + model="layers.layer", + source_pk=502, + uuid_value=shared_uuid, + fields=self._layer_fields("Name B"), + relations={}, + ), + ] + + with self.assertRaises(ValueError): + import_fixture_rows(fixture_rows, **self._import_kwargs()) + + def test_missing_relation_uuid_raises_error_under_strict_policy(self): + """Raise ValueError if required relations are missing from the fixture.""" + self._require_importer() + + assoc_uuid = uuid4() + missing_parent_uuid = uuid4() + missing_target_uuid = uuid4() + fixture_rows = [ + build_node( + model="layers.multilayerassociation", + source_pk=601, + uuid_value=assoc_uuid, + fields={"name": "Val-1aVal-2b"}, + relations={ + "parentLayer": build_ref( + model="layers.layer", + source_pk=9601, + uuid_value=missing_parent_uuid, + ), + "layer": build_ref( + model="layers.layer", + source_pk=9602, + uuid_value=missing_target_uuid, + ), + }, + ) + ] + + with self.assertRaises(ValueError): + import_fixture_rows(fixture_rows, **self._import_kwargs())