change(tests): include rdm and videos packages

cds_migrator_kit/rdm/affiliations/__init__.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM migration stats module."""
+
 from cds_dojson.overdo import OverdoBase
 
 affiliations_migrator_marc21 = OverdoBase(

cds_migrator_kit/rdm/affiliations/load.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM migration load module."""
+
 import json
 import logging
 import os

cds_migrator_kit/rdm/affiliations/streams.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM migration streams module."""
+
 from invenio_rdm_migrator.streams import StreamDefinition
 from invenio_rdm_migrator.transform import IdentityTransform
 

cds_migrator_kit/rdm/affiliations/transform.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM transform step module."""
+
 import logging
 from copy import deepcopy
 
@@ -44,7 +45,7 @@ def get_ror_affiliation(affiliation):
         except Exception as err:
             cli_logger.exception(err)
 
-    (chosen, affiliation) = get_ror_affiliation(affiliation_name)
+    chosen, affiliation = get_ror_affiliation(affiliation_name)
 
     return (chosen, affiliation)
 
@@ -76,7 +77,7 @@ def _affiliations(self, json_entry, key):
                     "original_input": affiliation_name,
                 }
 
-                (chosen, match_or_suggestions) = affiliations_search(affiliation_name)
+                chosen, match_or_suggestions = affiliations_search(affiliation_name)
 
                 if chosen:
                     _affiliation.update(

cds_migrator_kit/rdm/cli.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM command line module."""
+
 import logging
 from datetime import datetime
 from pathlib import Path

cds_migrator_kit/rdm/comments/load.py

@@ -69,6 +69,7 @@ def get_attached_files_for_comment(self, recid, comment_id):
         return []
 
     def get_oldest_record(self, parent_pid_value):
+        """Return the oldest version of a record by parent PID."""
         latest_record = current_rdm_records_service.read_latest(
             identity=system_identity, id_=parent_pid_value
         )

cds_migrator_kit/rdm/comments/log.py

@@ -35,9 +35,7 @@ def __init__(self, log_dir, collection=None):
         self.log_dir = Path(log_dir)
         self.collection = collection
         # Per-collection file logs + CSV; shared dir when collection is unset
-        self.files_dir = (
-            self.log_dir / collection if collection else self.log_dir
-        )
+        self.files_dir = self.log_dir / collection if collection else self.log_dir
         os.makedirs(self.files_dir, exist_ok=True)
 
         # Initializes logging format and file handlers for logging module (not CSV report).

cds_migrator_kit/rdm/comments/runner.py

@@ -36,8 +36,10 @@ def __init__(
         self.log_dir = Path(log_dir)
 
         self.logger = CommentsLogger(self.log_dir, collection)
-        collection_dirpath=collection_config["dir_path"]
-        comments_metadata_filepath = os.path.join(collection_dirpath, "comments_metadata.json")
+        collection_dirpath = collection_config["dir_path"]
+        comments_metadata_filepath = os.path.join(
+            collection_dirpath, "comments_metadata.json"
+        )
 
         self.stream = Stream(
             stream_definition.name,
@@ -71,7 +73,7 @@ def __init__(
         """Constructor."""
         config = read_config(config_filepath)
         collection_config = config["comments"][collection]
-        dirpath=collection_config["dir_path"]
+        dirpath = collection_config["dir_path"]
         missing_users_dir = os.path.join(dirpath, "users")
         filename = "missing_commentors_from_ldap.json"
 

cds_migrator_kit/rdm/comments/streams.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-Migrator-Kit comments streams module."""
+
 from invenio_rdm_migrator.streams import StreamDefinition
 from invenio_rdm_migrator.transform import IdentityTransform
 

cds_migrator_kit/rdm/records/load/entities/__init__.py

@@ -0,0 +1 @@
+"""CDS-RDM record load entities."""

cds_migrator_kit/rdm/records/load/entities/approval_request.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Validates an EP committee approval request's legacy history."""
+
 from datetime import datetime, timezone
 
 from cds_rdm.requests.committee_approval import APPRN_PID_TYPE
@@ -41,6 +42,7 @@ def __init__(
         resource_type=None,
         dry_run=False,
     ):
+        """Constructor."""
         self.ep_approval = ep_approval
         self.legacy_recid = legacy_recid
         self.title = title

cds_migrator_kit/rdm/records/load/entities/approval_request_load.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Creates and approves a migrated EP committee approval request."""
+
 from cds_rdm.requests.committee_approval import APPRN_PID_TYPE, CommitteeApprovalRequest
 from flask import current_app
 from invenio_access.permissions import system_identity
@@ -247,6 +248,4 @@ def _build_request(self, restricted_recid, restricted_parent, uow):
         self._create_reviewing_log_event(request, uow)
         self._apply_approved_entry(request, uow)
 
-        uow.register(
-            RecordCommitOp(request, indexer=current_requests_service.indexer)
-        )
+        uow.register(RecordCommitOp(request, indexer=current_requests_service.indexer))

cds_migrator_kit/rdm/records/load/entities/ep_split.py

@@ -58,6 +58,7 @@ class MetadataEntry:
     """Build a load entry for the public or restricted EP approval split."""
 
     def __init__(self, entry: MigrationEntry, approval_request, migration_logger):
+        """Constructor."""
         self.entry = entry
         self.approval_request = approval_request
         self.migration_logger = migration_logger
@@ -215,6 +216,7 @@ def _no_versions_error_message(self):
         return "No public files found to load for EP approval public split"
 
     def identifiers(self, identifiers):
+        """Return identifiers for the public split, dropping EP approval RNs."""
         kept = []
         removed = []
         for id_entry in identifiers:
@@ -317,6 +319,7 @@ def _no_versions_error_message(self):
         return "No files found to load for EP approval restricted split"
 
     def identifiers(self, identifiers):
+        """Return identifiers for the restricted split, keeping draft RNs."""
         kept = []
         removed = []
         for id_entry in identifiers:

cds_migrator_kit/rdm/records/load/entities/request.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Creates a community-inclusion request from a ``RecordRequest``."""
+
 import datetime
 
 from invenio_access.permissions import system_identity
@@ -30,7 +31,7 @@ class RequestLoad:
     ``ParentLoad``.
     """
 
-    def __init__(self, entry:MigrationEntry):
+    def __init__(self, entry: MigrationEntry):
         """Constructor.
 
         :param record_request: the built ``RecordRequest`` for this entry

cds_migrator_kit/rdm/records/streams.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM migration streams module."""
+
 from invenio_rdm_migrator.streams import StreamDefinition
 
 from cds_migrator_kit.extract.extract import LegacyExtract

cds_migrator_kit/rdm/records/transform/entities/__init__.py

@@ -0,0 +1 @@
+"""CDS-RDM record transform entities."""

cds_migrator_kit/rdm/records/transform/entities/parent.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """The RDM parent record for one migrated CDS record."""
+
 import re
 
 from flask import current_app

cds_migrator_kit/rdm/records/transform/entities/record.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """The RDM record's own content - ``MigrationEntry["record"]``."""
+
 from copy import deepcopy
 from typing import Any, TypedDict
 
@@ -226,8 +227,7 @@ def _verify_publication_date(self, raw_dump_entry, dojson_entry):
         creation date) and no creation date, raise an exception.
         """
         if not raw_dump_entry.get("files") and not (
-            dojson_entry.get("status_week_date")
-            or dojson_entry.get("publication_date")
+            dojson_entry.get("status_week_date") or dojson_entry.get("publication_date")
         ):
             raise ManualImportRequired(
                 message="Record missing publication date",

cds_migrator_kit/rdm/records/transform/entities/request.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """A community-inclusion request for one migrated CDS record."""
+
 from invenio_accounts.models import User
 from invenio_db import db
 
@@ -57,7 +58,6 @@ def __bool__(self):
         """True when there's request data to act on."""
         return bool(self.data)
 
-
     def _resolve_reviewers(self, reviewer_names):
         """Resolve raw reviewer name/email strings to RDM reviewer entries.
 

cds_migrator_kit/rdm/records/transform/mappers/base.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Base classes shared by all CDS-RDM record field mappers."""
+
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 

cds_migrator_kit/rdm/records/transform/mappers/contributors.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Creators/contributors field mapping: affiliations and person-id lookup."""
+
 from copy import deepcopy
 
 from idutils import normalize_ror
@@ -124,11 +125,7 @@ def _lookup_person_id(creator):
             user_id = ui.user.id
             names = NamesMetadata.query.filter_by(internal_id=str(user_id)).all()
             name = next(
-                (
-                    name
-                    for name in names
-                    if "unlisted" not in name.json.get("tags", [])
-                ),
+                (name for name in names if "unlisted" not in name.json.get("tags", [])),
                 None,
             )
     # filter out cern person_id

cds_migrator_kit/rdm/records/transform/mappers/metadata.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """``metadata`` field mappers for CDS to RDM record transformation."""
+
 from dateutil.parser import parse
 
 from cds_migrator_kit.errors import MissingRequiredField, UnexpectedValue

cds_migrator_kit/rdm/records/transform/mappers/record.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Top-level ``record_json_output`` field mappers."""
+
 from cds_migrator_kit.rdm.records.transform.mappers.base import FieldMapper
 
 

cds_migrator_kit/rdm/records/transform/mappers/registry.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Composed lists of field mappers used by RecordEntry."""
+
 from cds_migrator_kit.rdm.records.transform.mappers.base import (
     PassthroughCustomFieldMapper,
     PassthroughMapper,

cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """YAML-backed vocabulary lookup used by custom_fields mappers."""
+
 from pathlib import Path
 
 import yaml

cds_migrator_kit/rdm/records/transform/models/ab_dep.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM CMS note model."""
+
 from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
     rdm_base_publication_model,
 )
@@ -15,14 +16,14 @@
 class LEPResearchModel(ResearchModel):
     """Translation model for research."""
 
-    __query__ = '(710__.5:SI OR 710__.5:SC OR 710__.5:SL OR 710__.5:PS OR 710__.5:MPS OR 710__.5:ISR OR 710__.5:MSC OR 710__.5:AC OR 710__.5:SPS OR 710__.5:LEP OR 710__.5:AB OR 710__.5:AR) AND (980__:ARTICLE OR 980__:PREPRINT) OR 690C_.a:ISRRunning OR 690C_.a:ISRPerForm -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM -710__.5:TS -710__.5:ST -710__.5:MT -710__.5:EST -710__.5:SB -088:CERN-ALEPH-PUB-* -693__.e:ALEPH'
+    __query__ = "(710__.5:SI OR 710__.5:SC OR 710__.5:SL OR 710__.5:PS OR 710__.5:MPS OR 710__.5:ISR OR 710__.5:MSC OR 710__.5:AC OR 710__.5:SPS OR 710__.5:LEP OR 710__.5:AB OR 710__.5:AR) AND (980__:ARTICLE OR 980__:PREPRINT) OR 690C_.a:ISRRunning OR 690C_.a:ISRPerForm -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM -710__.5:TS -710__.5:ST -710__.5:MT -710__.5:EST -710__.5:SB -088:CERN-ALEPH-PUB-* -693__.e:ALEPH"
 
     __ignore_keys__ = {
-        "594__a", # can be ignored for this collection
-        "775__p", # can be ignored for this collection - title of another volume
-        "775__c", # year of volume
-        "596__a", # multivolume tag
-        "300__x", # drop the physical description
+        "594__a",  # can be ignored for this collection
+        "775__p",  # can be ignored for this collection - title of another volume
+        "775__c",  # year of volume
+        "596__a",  # multivolume tag
+        "300__x",  # drop the physical description
         "8564_z",  # file comment, migrated via file metadata
         "0248_a",
         "0248_p",

cds_migrator_kit/rdm/records/transform/models/annual_report.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM CMS note model."""
+
 from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
     rdm_base_publication_model,
 )

cds_migrator_kit/rdm/records/transform/models/antares.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM CMS note model."""
+
 from cds_migrator_kit.rdm.records.transform.models.research import (
     research_model,
 )
@@ -18,7 +19,7 @@ class ANTARESResearchModel(CdsOverdo):
     __query__ = '980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'
 
     __ignore_keys__ = {
-        "923__r", # place of photo, present in posters but redundant
+        "923__r",  # place of photo, present in posters but redundant
         "0248_a",
         "0248_p",
         "0248_q",

cds_migrator_kit/rdm/records/transform/models/at.py

@@ -5,6 +5,8 @@
 # CDS-RDM is free software; you can redistribute it and/or modify it under
 # the terms of the MIT License; see LICENSE file for more details.
 
+"""CDS-RDM AT migration model."""
+
 from cds_migrator_kit.rdm.records.transform.models._config import IGNORE_SYSTEM_KEYS
 from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
     rdm_base_publication_model,

cds_migrator_kit/rdm/records/transform/models/base_publication_record.py

@@ -1,3 +1,5 @@
+"""CDS-RDM base publication migration model."""
+
 from cds_migrator_kit.rdm.records.transform.models.base_record import (
     rdm_base_record_model,
 )

cds_migrator_kit/rdm/records/transform/models/beams.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM Beams model."""
+
 from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
     rdm_base_publication_model,
 )

cds_migrator_kit/rdm/records/transform/models/books.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM Yellow report model."""
+
 from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
     rdm_base_publication_model,
 )

cds_migrator_kit/rdm/records/transform/models/bulletin_issue.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM CMS note model."""
+
 from cds_migrator_kit.rdm.records.transform.models.base_record import (
     rdm_base_record_model,
 )

cds_migrator_kit/rdm/records/transform/models/courier.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM CMS note model."""
+
 from cds_migrator_kit.rdm.records.transform.models.base_record import (
     rdm_base_record_model,
 )