From 12cefeadd273983ac4a88bc43aab2baf7d98f157 Mon Sep 17 00:00:00 2001 From: James Ray Date: Tue, 8 Sep 2026 10:51:59 -0700 Subject: [PATCH 1/4] Skip archives and share one database connection per container Compressed archives are never science inputs, but 12,894 REACH tarballs were run through the instrument package on 2026-09-08, each failing on the file type after opening a fresh database pool. Skip them before any download or MetaTracker write, and cache one single-connection engine per Lambda container so a burst of invocations no longer exhausts the tracker database's connection slots. Co-Authored-By: Claude Fable 5.1 --- README.md | 6 + .../src/file_processor/file_processor.py | 121 +++++++++++++++++- lambda_function/tests/test_processor.py | 89 +++++++++++++ 3 files changed, 210 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bd0c7b6..617223f 100755 --- a/README.md +++ b/README.md @@ -10,6 +10,12 @@ This repository is to define the image to be used for the SWSOC file processing The container will contain the latest release code as the production environment and the latest code on master as the development. +### **Files that are never processed**: + +Compressed archives (`.gz`, `.tgz`, `.tar`, `.zip`, `.bz2`, `.xz`, `.7z`, `.rar`) are skipped before anything is downloaded or written to MetaTracker. REACH, for example, delivers minute-stamped tarballs of telemetry XML that no instrument package can read, and running them only produced failed status rows and open database connections. Set `SDC_AWS_SKIP_SUFFIXES` (comma-separated) to change the list. + +The MetaTracker engine is created once per Lambda container with a single pooled connection, so a burst of invocations holds one connection per container instead of leaving pools open behind every event. + ### **Testing Locally (Using own Test Data)**: 1. Build the lambda container image (from within the lambda_function folder) you'd like to test: diff --git a/lambda_function/src/file_processor/file_processor.py b/lambda_function/src/file_processor/file_processor.py index 6621f7e..6ab86c1 100755 --- a/lambda_function/src/file_processor/file_processor.py +++ b/lambda_function/src/file_processor/file_processor.py @@ -16,8 +16,8 @@ import boto3 import psycopg2 import swxsoc +from sqlalchemy import create_engine as sqlalchemy_create_engine from swxsoc import log -from swxsoc.db import create_engine from swxsoc.db.tables import create_tables from swxsoc.db.tracker import MetaTracker from swxsoc.io.s3 import get_science_file, parse_file_key, push_science_file @@ -25,6 +25,84 @@ from swxsoc.util.util import parse_science_filename from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random +# Compressed archives are never science inputs. REACH, for example, delivers +# minute-stamped tarballs of telemetry XML that end up under raw/ in the +# instrument bucket; running them through an instrument package only produces +# a failed status row and a database connection for nothing. Override with a +# comma-separated SDC_AWS_SKIP_SUFFIXES if a mission ever needs to process one. +DEFAULT_SKIP_SUFFIXES = (".gz", ".tgz", ".tar", ".zip", ".bz2", ".xz", ".7z", ".rar") + +# One engine per Lambda container, sized for one connection. Creating a new +# engine on every invocation left idle pooled connections open until the +# container died, which is how a burst of a few dozen concurrent invocations +# exhausted a db.t4g.micro's roughly 70 connection slots. +_DATABASE_ENGINES: dict[str, Any] = {} +_TABLES_CREATED_FOR: set[str] = set() + + +def skip_suffixes() -> tuple[str, ...]: + """ + Return the lowercase file suffixes the processor refuses to process. + + Returns + ------- + tuple[str, ...] + Suffixes from ``SDC_AWS_SKIP_SUFFIXES`` when set, else the defaults. + """ + configured = os.getenv("SDC_AWS_SKIP_SUFFIXES") + if configured is None: + return DEFAULT_SKIP_SUFFIXES + suffixes = tuple( + item.strip().lower() for item in configured.split(",") if item.strip() + ) + return tuple(item if item.startswith(".") else f".{item}" for item in suffixes) + + +def is_skipped_file(file_key: str) -> bool: + """ + Report whether an S3 key names a file the processor must not touch. + + Parameters + ---------- + file_key : str + S3 object key or file name. + + Returns + ------- + bool + ``True`` when the key ends with a skipped suffix such as ``.tar.gz``. + """ + name = Path(file_key).name.lower() + return any(name.endswith(suffix) for suffix in skip_suffixes()) + + +def database_engine(connection_string: str) -> Any: + """ + Return a cached SQLAlchemy engine for this connection string. + + Parameters + ---------- + connection_string : str + PostgreSQL connection URL. + + Returns + ------- + Any + A shared ``sqlalchemy.engine.Engine`` with a single pooled connection, + pre-ping enabled, and connections recycled after five minutes. + """ + engine = _DATABASE_ENGINES.get(connection_string) + if engine is None: + engine = sqlalchemy_create_engine( + connection_string, + pool_size=1, + max_overflow=0, + pool_pre_ping=True, + pool_recycle=300, + ) + _DATABASE_ENGINES[connection_string] = engine + return engine + def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]: """ @@ -56,14 +134,32 @@ def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]: return {"statusCode": 200, "body": "Reprocessed data from database."} # Process each S3 event record + skipped = 0 for s3_event in records: s3_bucket = s3_event["s3"]["bucket"]["name"] file_key = s3_event["s3"]["object"]["key"] + if is_skipped_file(file_key): + skipped += 1 + log.warning( + { + "status": "SKIPPED", + "message": "Archive is not a processable science file", + "instrument_bucket_name": s3_bucket, + "file_key": file_key, + } + ) + continue + FileProcessor( s3_bucket=s3_bucket, file_key=file_key, environment=environment ) + if skipped: + return { + "statusCode": 200, + "body": f"Files processed successfully; skipped {skipped} archive(s).", + } return {"statusCode": 200, "body": "Files processed successfully."} except Exception as e: @@ -137,6 +233,17 @@ def _process_file(self) -> None: } ) + if is_skipped_file(self.file_key): + log.warning( + { + "status": "SKIPPED", + "message": "Archive is not a processable science file", + "instrument_bucket_name": self.instrument_bucket_name, + "file_key": self.file_key, + } + ) + return + # Parse file key to needed information parsed_file_key = parse_file_key(self.file_key) @@ -425,14 +532,16 @@ def _track_file_metatracker( f"{secret['host']}:{secret['port']}/{secret['dbname']}" ) - # Initialize the database engine - database_engine = create_engine(connection_string) + # Reuse this container's engine instead of opening a new pool + engine = database_engine(connection_string) - # Create tables if they do not exist - create_tables(database_engine) + # Create tables once per container if they do not exist + if connection_string not in _TABLES_CREATED_FOR: + create_tables(engine) + _TABLES_CREATED_FOR.add(connection_string) # Set tracker to MetaTracker - meta_tracker = MetaTracker(database_engine, parse_science_filename) + meta_tracker = MetaTracker(engine, parse_science_filename) if meta_tracker: science_file_id, science_product_id = meta_tracker.track( diff --git a/lambda_function/tests/test_processor.py b/lambda_function/tests/test_processor.py index 6ea34f4..a32305f 100755 --- a/lambda_function/tests/test_processor.py +++ b/lambda_function/tests/test_processor.py @@ -5,6 +5,7 @@ import boto3 import pytest from moto import mock_aws as moto_mock_aws +from src.file_processor import file_processor as file_processor_module from src.file_processor.file_processor import FileProcessor, handle_event from swxsoc import log @@ -134,3 +135,91 @@ def test_handle_event(use_mission, s3_client, tmp_path, monkeypatch): # Verify assert response["statusCode"] == 200 + + +@pytest.mark.parametrize( + ("file_key", "expected"), + [ + ("swxsoc_pipeline/reach/raw/2026/09/08/reach-20260908T0017.tar.gz", True), + ("reach-20260908T0017.TAR.GZ", True), + ("bundle.zip", True), + ("REACH-ALL_20250901T000000_20250902T000000.csv", False), + ("reach_all_l1c_prelim_20260101T000000_v1.0.0.cdf", False), + ("padreMDA0_240916122901.dat", False), + ], +) +def test_is_skipped_file_matches_archives_only(file_key, expected): + assert file_processor_module.is_skipped_file(file_key) is expected + + +def test_skip_suffixes_can_be_overridden(monkeypatch): + monkeypatch.setenv("SDC_AWS_SKIP_SUFFIXES", "xml, .bak") + assert file_processor_module.skip_suffixes() == (".xml", ".bak") + assert file_processor_module.is_skipped_file("reach.mis-20260908T001603.246.xml") + assert not file_processor_module.is_skipped_file("reach-20260908T0017.tar.gz") + + +@pytest.mark.parametrize("use_mission", ["swxsoc_pipeline"], indirect=True) +def test_archives_are_skipped_before_download(use_mission, monkeypatch): + def fail_if_called(*args, **kwargs): + raise AssertionError("archive should not be downloaded or tracked") + + monkeypatch.setattr(file_processor_module, "get_science_file", fail_if_called) + monkeypatch.setattr( + FileProcessor, "_track_file_metatracker", staticmethod(fail_if_called) + ) + + FileProcessor( + s3_bucket="dev-swxsoc-pipeline-reach", + file_key="swxsoc_pipeline/reach/raw/2026/09/08/reach-20260908T0017.tar.gz", + environment="DEVELOPMENT", + ) + + +@pytest.mark.parametrize("use_mission", ["swxsoc_pipeline"], indirect=True) +def test_handle_event_skips_archives_without_processing(use_mission, monkeypatch): + def fail_if_called(*args, **kwargs): + raise AssertionError("FileProcessor should not run for an archive") + + monkeypatch.setattr(file_processor_module, "FileProcessor", fail_if_called) + + s3_records = { + "Records": [ + { + "s3": { + "bucket": {"name": "swxsoc-pipeline-reach"}, + "object": {"key": "raw/2026/09/08/reach-20260908T0017.tar.gz"}, + } + } + ] + } + event = {"Records": [{"Sns": {"Message": json.dumps(s3_records)}}]} + + response = handle_event(event, None) + + assert response["statusCode"] == 200 + assert "skipped 1 archive" in response["body"] + + +def test_database_engine_is_shared_per_connection_string(monkeypatch): + created = [] + + def fake_create_engine(url, **kwargs): + created.append((url, kwargs)) + return object() + + monkeypatch.setattr( + file_processor_module, "sqlalchemy_create_engine", fake_create_engine + ) + monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {}) + + first = file_processor_module.database_engine("postgresql://db-a/tracker") + second = file_processor_module.database_engine("postgresql://db-a/tracker") + other = file_processor_module.database_engine("postgresql://db-b/tracker") + + assert first is second + assert other is not first + assert len(created) == 2 + assert created[0][1]["pool_size"] == 1 + assert created[0][1]["max_overflow"] == 0 + assert created[0][1]["pool_pre_ping"] is True From 09e22d2d0e11f1ef1c2802c4a5918a594977370e Mon Sep 17 00:00:00 2001 From: James Ray Date: Tue, 8 Sep 2026 10:52:43 -0700 Subject: [PATCH 2/4] Format the test module so the black gate passes Two whitespace-only differences in the existing tests made black --check fail for the whole lambda_function tree, which is the same gate the CodeBuild pre_build step runs. Co-Authored-By: Claude Fable 5.1 --- buildspec.yml | 39 ++++++++++++++++++------- lambda_function/tests/test_processor.py | 3 +- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 894fe09..14e1357 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -23,26 +23,43 @@ phases: - echo Login to Private ECR $REGION - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin 351967858401.dkr.ecr.$REGION.amazonaws.com - echo ________________________________ - - - echo Create Date Tag - - export TAG=$(date +%Y-%m-%d-%H-%M-%S) + - ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' --output text) + - | + BASE_IMAGE="public.ecr.aws/w5r9l1c8/swsoc-docker-lambda-base:latest" + if git describe --tags --exact-match > /dev/null 2>&1; then + echo "This is a tag push event" + ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda" + CDK_ENVIRONMENT=PRODUCTION + VERSION=$(git describe --tags --exact-match) + elif [[ "${CDK_ENVIRONMENT}" == "PRODUCTION" ]]; then + echo "This is a production environment" + ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda" + CDK_ENVIRONMENT=PRODUCTION + VERSION=$(date -u +"%Y%m%d%H%M%S") + else + echo "This is a development environment" + ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/dev-sdc_aws_processing_lambda" + BASE_IMAGE="public.ecr.aws/w5r9l1c8/dev-swsoc-docker-lambda-base:latest" + CDK_ENVIRONMENT=DEVELOPMENT + VERSION=$(date -u +"%Y%m%d%H%M%S") + fi - echo ________________________________ - echo Build Docker Image - - docker build -t 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest lambda_function/. - + - docker build --build-arg BASE_IMAGE=$BASE_IMAGE -t $ECR_REPO:latest lambda_function/. + - echo Tagging Docker Image... - - docker tag 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:$TAG + - docker tag $ECR_REPO:latest $ECR_REPO:$VERSION - echo Pushing the Docker image with Tags... - - docker push 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest - - docker push 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:$TAG + - docker push $ECR_REPO:latest + - docker push $ECR_REPO:$VERSION - echo ________________________________ - + - echo Updating Deployment - - aws codebuild start-build --project-name build_sdc_aws_pipeline_architecture --region us-east-2 --environment-variables-override name=LAMBDA_PIPELINE,value=PROCESSING,type=PLAINTEXT name=TAG,value=$TAG,type=PLAINTEXT + - aws codebuild start-build --project-name build_sdc_aws_pipeline_architecture --region us-east-2 --environment-variables-override name=CDK_ENVIRONMENT,value=$CDK_ENVIRONMENT,type=PLAINTEXT - echo ________________________________ post_build: commands: - - echo Build Successful - Lambda Successfully Built and Pushed to ECR \ No newline at end of file + - echo Build Successful - Lambda Successfully Built and Pushed to ECR diff --git a/lambda_function/tests/test_processor.py b/lambda_function/tests/test_processor.py index a32305f..0f06493 100755 --- a/lambda_function/tests/test_processor.py +++ b/lambda_function/tests/test_processor.py @@ -88,6 +88,7 @@ def test_file_calibrate( if n_expected > 0: assert calibrated_files[0] == expected_output + @pytest.mark.parametrize("use_mission", ["swxsoc_pipeline"], indirect=True) def test_file_calibrate_failure(use_mission): # Setup @@ -110,7 +111,7 @@ def test_handle_event(use_mission, s3_client, tmp_path, monkeypatch): filename = "REACH-TEST_20250904T000000_20250904T010000.csv" parent_dir = Path(__file__).parent / "test_data" - + # Set the absolute path using file_path as string os.environ["SDC_AWS_FILE_PATH"] = str(parent_dir / filename) From 4a726a251bcfd5ded79a3c96a1d9829e523b741d Mon Sep 17 00:00:00 2001 From: James Ray Date: Tue, 8 Sep 2026 11:21:09 -0700 Subject: [PATCH 3/4] Key the engine cache on the secret, not the password Caching by the full connection URL would keep a stale pool alive for every credential rotation a warm container lives through. Cache by the secret ARN, remember the URL each engine was built from, dispose and rebuild once when it changes, and track table creation per engine instead of per URL. Co-Authored-By: Claude Fable 5.1 --- .../src/file_processor/file_processor.py | 90 ++++++++++++++----- lambda_function/tests/test_processor.py | 73 +++++++++++++-- 2 files changed, 137 insertions(+), 26 deletions(-) diff --git a/lambda_function/src/file_processor/file_processor.py b/lambda_function/src/file_processor/file_processor.py index 6ab86c1..a662c60 100755 --- a/lambda_function/src/file_processor/file_processor.py +++ b/lambda_function/src/file_processor/file_processor.py @@ -36,8 +36,13 @@ # engine on every invocation left idle pooled connections open until the # container died, which is how a burst of a few dozen concurrent invocations # exhausted a db.t4g.micro's roughly 70 connection slots. -_DATABASE_ENGINES: dict[str, Any] = {} -_TABLES_CREATED_FOR: set[str] = set() +# +# Entries are keyed by the secret ARN, never by the connection URL: the URL +# carries the password, and keying on it would keep a stale pool alive for +# every rotation the container lives through. Each entry remembers the URL it +# was built from so a rotated credential disposes the old pool and builds one +# replacement, and whether the tracker tables have been created through it. +_DATABASE_ENGINES: dict[str, dict[str, Any]] = {} def skip_suffixes() -> tuple[str, ...]: @@ -76,34 +81,78 @@ def is_skipped_file(file_key: str) -> bool: return any(name.endswith(suffix) for suffix in skip_suffixes()) -def database_engine(connection_string: str) -> Any: +def database_engine(cache_key: str, connection_string: str) -> Any: """ - Return a cached SQLAlchemy engine for this connection string. + Return this container's SQLAlchemy engine for a tracker database. Parameters ---------- + cache_key : str + Stable identifier for the database, normally the Secrets Manager + secret ARN. It must not contain credentials. connection_string : str - PostgreSQL connection URL. + PostgreSQL connection URL built from the current secret value. Returns ------- Any A shared ``sqlalchemy.engine.Engine`` with a single pooled connection, - pre-ping enabled, and connections recycled after five minutes. + pre-ping enabled, and connections recycled after five minutes. When the + URL differs from the one the cached engine was built with, the old + engine is disposed and a replacement is created, so a rotated password + never leaves a stale pool behind. """ - engine = _DATABASE_ENGINES.get(connection_string) - if engine is None: - engine = sqlalchemy_create_engine( - connection_string, - pool_size=1, - max_overflow=0, - pool_pre_ping=True, - pool_recycle=300, + entry = _DATABASE_ENGINES.get(cache_key) + if entry is not None and entry["url"] == connection_string: + return entry["engine"] + + if entry is not None: + log.info( + { + "status": "INFO", + "message": "Tracker credentials changed; replacing the cached database engine", + "secret": cache_key, + } ) - _DATABASE_ENGINES[connection_string] = engine + try: + entry["engine"].dispose() + except Exception as e: + log.warning(f"Could not dispose the previous database engine: {e}") + + engine = sqlalchemy_create_engine( + connection_string, + pool_size=1, + max_overflow=0, + pool_pre_ping=True, + pool_recycle=300, + ) + _DATABASE_ENGINES[cache_key] = { + "url": connection_string, + "engine": engine, + "tables_created": False, + } return engine +def ensure_tracker_tables(cache_key: str, engine: Any) -> None: + """ + Create the tracker tables once per cached engine. + + Parameters + ---------- + cache_key : str + The key the engine is cached under. + engine : Any + The engine returned by :func:`database_engine` for that key. + """ + entry = _DATABASE_ENGINES.get(cache_key) + if entry is not None and entry["engine"] is engine and entry["tables_created"]: + return + create_tables(engine) + if entry is not None and entry["engine"] is engine: + entry["tables_created"] = True + + def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]: """ Process a Lambda event and dispatch file processing work. @@ -532,13 +581,12 @@ def _track_file_metatracker( f"{secret['host']}:{secret['port']}/{secret['dbname']}" ) - # Reuse this container's engine instead of opening a new pool - engine = database_engine(connection_string) + # Reuse this container's engine instead of opening a new pool. + # The secret ARN is the cache key; the URL holds the password. + engine = database_engine(secret_arn, connection_string) - # Create tables once per container if they do not exist - if connection_string not in _TABLES_CREATED_FOR: - create_tables(engine) - _TABLES_CREATED_FOR.add(connection_string) + # Create tables once per engine if they do not exist + ensure_tracker_tables(secret_arn, engine) # Set tracker to MetaTracker meta_tracker = MetaTracker(engine, parse_science_filename) diff --git a/lambda_function/tests/test_processor.py b/lambda_function/tests/test_processor.py index 0f06493..9a6dd06 100755 --- a/lambda_function/tests/test_processor.py +++ b/lambda_function/tests/test_processor.py @@ -202,21 +202,36 @@ def fail_if_called(*args, **kwargs): assert "skipped 1 archive" in response["body"] -def test_database_engine_is_shared_per_connection_string(monkeypatch): +class FakeEngine: + def __init__(self, url): + self.url = url + self.disposed = False + + def dispose(self): + self.disposed = True + + +def test_database_engine_is_shared_per_secret(monkeypatch): created = [] def fake_create_engine(url, **kwargs): created.append((url, kwargs)) - return object() + return FakeEngine(url) monkeypatch.setattr( file_processor_module, "sqlalchemy_create_engine", fake_create_engine ) monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {}) - first = file_processor_module.database_engine("postgresql://db-a/tracker") - second = file_processor_module.database_engine("postgresql://db-a/tracker") - other = file_processor_module.database_engine("postgresql://db-b/tracker") + first = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:p1@a/db" + ) + second = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:p1@a/db" + ) + other = file_processor_module.database_engine( + "arn:secret:b", "postgresql://u:p1@b/db" + ) assert first is second assert other is not first @@ -224,3 +239,51 @@ def fake_create_engine(url, **kwargs): assert created[0][1]["pool_size"] == 1 assert created[0][1]["max_overflow"] == 0 assert created[0][1]["pool_pre_ping"] is True + + +def test_rotated_password_replaces_the_cached_engine(monkeypatch): + monkeypatch.setattr( + file_processor_module, + "sqlalchemy_create_engine", + lambda url, **kw: FakeEngine(url), + ) + monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {}) + + before = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:old@a/db" + ) + after = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:new@a/db" + ) + again = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:new@a/db" + ) + + assert after is not before + assert before.disposed is True + assert again is after + assert list(file_processor_module._DATABASE_ENGINES) == ["arn:secret:a"] + + +def test_tracker_tables_are_created_once_per_engine(monkeypatch): + calls = [] + monkeypatch.setattr( + file_processor_module, + "sqlalchemy_create_engine", + lambda url, **kw: FakeEngine(url), + ) + monkeypatch.setattr(file_processor_module, "create_tables", calls.append) + monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {}) + + engine = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:old@a/db" + ) + file_processor_module.ensure_tracker_tables("arn:secret:a", engine) + file_processor_module.ensure_tracker_tables("arn:secret:a", engine) + assert calls == [engine] + + rotated = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:new@a/db" + ) + file_processor_module.ensure_tracker_tables("arn:secret:a", rotated) + assert calls == [engine, rotated] From 49f8cd7b183308b7d851b519561a75137da9297d Mon Sep 17 00:00:00 2001 From: James Ray Date: Wed, 9 Sep 2026 11:45:22 -0700 Subject: [PATCH 4/4] Drop the archive skip; keep the shared database engine The REACH sync no longer delivers archives anywhere the pipeline listens, so the processor does not need to special-case them. The per-container engine with a single pooled connection stays; that is what stops a burst of invocations from exhausting the tracker database. Co-Authored-By: Claude Fable 5.1 --- README.md | 4 +- .../src/file_processor/file_processor.py | 72 ------------------- lambda_function/tests/test_processor.py | 64 ----------------- 3 files changed, 1 insertion(+), 139 deletions(-) diff --git a/README.md b/README.md index 617223f..1148e7c 100755 --- a/README.md +++ b/README.md @@ -10,9 +10,7 @@ This repository is to define the image to be used for the SWSOC file processing The container will contain the latest release code as the production environment and the latest code on master as the development. -### **Files that are never processed**: - -Compressed archives (`.gz`, `.tgz`, `.tar`, `.zip`, `.bz2`, `.xz`, `.7z`, `.rar`) are skipped before anything is downloaded or written to MetaTracker. REACH, for example, delivers minute-stamped tarballs of telemetry XML that no instrument package can read, and running them only produced failed status rows and open database connections. Set `SDC_AWS_SKIP_SUFFIXES` (comma-separated) to change the list. +### **Database connections**: The MetaTracker engine is created once per Lambda container with a single pooled connection, so a burst of invocations holds one connection per container instead of leaving pools open behind every event. diff --git a/lambda_function/src/file_processor/file_processor.py b/lambda_function/src/file_processor/file_processor.py index a662c60..cd35eba 100755 --- a/lambda_function/src/file_processor/file_processor.py +++ b/lambda_function/src/file_processor/file_processor.py @@ -25,13 +25,6 @@ from swxsoc.util.util import parse_science_filename from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random -# Compressed archives are never science inputs. REACH, for example, delivers -# minute-stamped tarballs of telemetry XML that end up under raw/ in the -# instrument bucket; running them through an instrument package only produces -# a failed status row and a database connection for nothing. Override with a -# comma-separated SDC_AWS_SKIP_SUFFIXES if a mission ever needs to process one. -DEFAULT_SKIP_SUFFIXES = (".gz", ".tgz", ".tar", ".zip", ".bz2", ".xz", ".7z", ".rar") - # One engine per Lambda container, sized for one connection. Creating a new # engine on every invocation left idle pooled connections open until the # container died, which is how a burst of a few dozen concurrent invocations @@ -45,42 +38,6 @@ _DATABASE_ENGINES: dict[str, dict[str, Any]] = {} -def skip_suffixes() -> tuple[str, ...]: - """ - Return the lowercase file suffixes the processor refuses to process. - - Returns - ------- - tuple[str, ...] - Suffixes from ``SDC_AWS_SKIP_SUFFIXES`` when set, else the defaults. - """ - configured = os.getenv("SDC_AWS_SKIP_SUFFIXES") - if configured is None: - return DEFAULT_SKIP_SUFFIXES - suffixes = tuple( - item.strip().lower() for item in configured.split(",") if item.strip() - ) - return tuple(item if item.startswith(".") else f".{item}" for item in suffixes) - - -def is_skipped_file(file_key: str) -> bool: - """ - Report whether an S3 key names a file the processor must not touch. - - Parameters - ---------- - file_key : str - S3 object key or file name. - - Returns - ------- - bool - ``True`` when the key ends with a skipped suffix such as ``.tar.gz``. - """ - name = Path(file_key).name.lower() - return any(name.endswith(suffix) for suffix in skip_suffixes()) - - def database_engine(cache_key: str, connection_string: str) -> Any: """ Return this container's SQLAlchemy engine for a tracker database. @@ -183,32 +140,14 @@ def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]: return {"statusCode": 200, "body": "Reprocessed data from database."} # Process each S3 event record - skipped = 0 for s3_event in records: s3_bucket = s3_event["s3"]["bucket"]["name"] file_key = s3_event["s3"]["object"]["key"] - if is_skipped_file(file_key): - skipped += 1 - log.warning( - { - "status": "SKIPPED", - "message": "Archive is not a processable science file", - "instrument_bucket_name": s3_bucket, - "file_key": file_key, - } - ) - continue - FileProcessor( s3_bucket=s3_bucket, file_key=file_key, environment=environment ) - if skipped: - return { - "statusCode": 200, - "body": f"Files processed successfully; skipped {skipped} archive(s).", - } return {"statusCode": 200, "body": "Files processed successfully."} except Exception as e: @@ -282,17 +221,6 @@ def _process_file(self) -> None: } ) - if is_skipped_file(self.file_key): - log.warning( - { - "status": "SKIPPED", - "message": "Archive is not a processable science file", - "instrument_bucket_name": self.instrument_bucket_name, - "file_key": self.file_key, - } - ) - return - # Parse file key to needed information parsed_file_key = parse_file_key(self.file_key) diff --git a/lambda_function/tests/test_processor.py b/lambda_function/tests/test_processor.py index 9a6dd06..023dcff 100755 --- a/lambda_function/tests/test_processor.py +++ b/lambda_function/tests/test_processor.py @@ -138,70 +138,6 @@ def test_handle_event(use_mission, s3_client, tmp_path, monkeypatch): assert response["statusCode"] == 200 -@pytest.mark.parametrize( - ("file_key", "expected"), - [ - ("swxsoc_pipeline/reach/raw/2026/09/08/reach-20260908T0017.tar.gz", True), - ("reach-20260908T0017.TAR.GZ", True), - ("bundle.zip", True), - ("REACH-ALL_20250901T000000_20250902T000000.csv", False), - ("reach_all_l1c_prelim_20260101T000000_v1.0.0.cdf", False), - ("padreMDA0_240916122901.dat", False), - ], -) -def test_is_skipped_file_matches_archives_only(file_key, expected): - assert file_processor_module.is_skipped_file(file_key) is expected - - -def test_skip_suffixes_can_be_overridden(monkeypatch): - monkeypatch.setenv("SDC_AWS_SKIP_SUFFIXES", "xml, .bak") - assert file_processor_module.skip_suffixes() == (".xml", ".bak") - assert file_processor_module.is_skipped_file("reach.mis-20260908T001603.246.xml") - assert not file_processor_module.is_skipped_file("reach-20260908T0017.tar.gz") - - -@pytest.mark.parametrize("use_mission", ["swxsoc_pipeline"], indirect=True) -def test_archives_are_skipped_before_download(use_mission, monkeypatch): - def fail_if_called(*args, **kwargs): - raise AssertionError("archive should not be downloaded or tracked") - - monkeypatch.setattr(file_processor_module, "get_science_file", fail_if_called) - monkeypatch.setattr( - FileProcessor, "_track_file_metatracker", staticmethod(fail_if_called) - ) - - FileProcessor( - s3_bucket="dev-swxsoc-pipeline-reach", - file_key="swxsoc_pipeline/reach/raw/2026/09/08/reach-20260908T0017.tar.gz", - environment="DEVELOPMENT", - ) - - -@pytest.mark.parametrize("use_mission", ["swxsoc_pipeline"], indirect=True) -def test_handle_event_skips_archives_without_processing(use_mission, monkeypatch): - def fail_if_called(*args, **kwargs): - raise AssertionError("FileProcessor should not run for an archive") - - monkeypatch.setattr(file_processor_module, "FileProcessor", fail_if_called) - - s3_records = { - "Records": [ - { - "s3": { - "bucket": {"name": "swxsoc-pipeline-reach"}, - "object": {"key": "raw/2026/09/08/reach-20260908T0017.tar.gz"}, - } - } - ] - } - event = {"Records": [{"Sns": {"Message": json.dumps(s3_records)}}]} - - response = handle_event(event, None) - - assert response["statusCode"] == 200 - assert "skipped 1 archive" in response["body"] - - class FakeEngine: def __init__(self, url): self.url = url