diff --git a/README.md b/README.md index bd0c7b6..1148e7c 100755 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ This repository is to define the image to be used for the SWSOC file processing The container will contain the latest release code as the production environment and the latest code on master as the development. +### **Database connections**: + +The MetaTracker engine is created once per Lambda container with a single pooled connection, so a burst of invocations holds one connection per container instead of leaving pools open behind every event. + ### **Testing Locally (Using own Test Data)**: 1. Build the lambda container image (from within the lambda_function folder) you'd like to test: diff --git a/buildspec.yml b/buildspec.yml index 894fe09..14e1357 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -23,26 +23,43 @@ phases: - echo Login to Private ECR $REGION - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin 351967858401.dkr.ecr.$REGION.amazonaws.com - echo ________________________________ - - - echo Create Date Tag - - export TAG=$(date +%Y-%m-%d-%H-%M-%S) + - ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' --output text) + - | + BASE_IMAGE="public.ecr.aws/w5r9l1c8/swsoc-docker-lambda-base:latest" + if git describe --tags --exact-match > /dev/null 2>&1; then + echo "This is a tag push event" + ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda" + CDK_ENVIRONMENT=PRODUCTION + VERSION=$(git describe --tags --exact-match) + elif [[ "${CDK_ENVIRONMENT}" == "PRODUCTION" ]]; then + echo "This is a production environment" + ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda" + CDK_ENVIRONMENT=PRODUCTION + VERSION=$(date -u +"%Y%m%d%H%M%S") + else + echo "This is a development environment" + ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/dev-sdc_aws_processing_lambda" + BASE_IMAGE="public.ecr.aws/w5r9l1c8/dev-swsoc-docker-lambda-base:latest" + CDK_ENVIRONMENT=DEVELOPMENT + VERSION=$(date -u +"%Y%m%d%H%M%S") + fi - echo ________________________________ - echo Build Docker Image - - docker build -t 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest lambda_function/. - + - docker build --build-arg BASE_IMAGE=$BASE_IMAGE -t $ECR_REPO:latest lambda_function/. + - echo Tagging Docker Image... - - docker tag 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:$TAG + - docker tag $ECR_REPO:latest $ECR_REPO:$VERSION - echo Pushing the Docker image with Tags... - - docker push 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest - - docker push 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:$TAG + - docker push $ECR_REPO:latest + - docker push $ECR_REPO:$VERSION - echo ________________________________ - + - echo Updating Deployment - - aws codebuild start-build --project-name build_sdc_aws_pipeline_architecture --region us-east-2 --environment-variables-override name=LAMBDA_PIPELINE,value=PROCESSING,type=PLAINTEXT name=TAG,value=$TAG,type=PLAINTEXT + - aws codebuild start-build --project-name build_sdc_aws_pipeline_architecture --region us-east-2 --environment-variables-override name=CDK_ENVIRONMENT,value=$CDK_ENVIRONMENT,type=PLAINTEXT - echo ________________________________ post_build: commands: - - echo Build Successful - Lambda Successfully Built and Pushed to ECR \ No newline at end of file + - echo Build Successful - Lambda Successfully Built and Pushed to ECR diff --git a/lambda_function/src/file_processor/file_processor.py b/lambda_function/src/file_processor/file_processor.py index 6621f7e..cd35eba 100755 --- a/lambda_function/src/file_processor/file_processor.py +++ b/lambda_function/src/file_processor/file_processor.py @@ -16,8 +16,8 @@ import boto3 import psycopg2 import swxsoc +from sqlalchemy import create_engine as sqlalchemy_create_engine from swxsoc import log -from swxsoc.db import create_engine from swxsoc.db.tables import create_tables from swxsoc.db.tracker import MetaTracker from swxsoc.io.s3 import get_science_file, parse_file_key, push_science_file @@ -25,6 +25,90 @@ from swxsoc.util.util import parse_science_filename from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random +# One engine per Lambda container, sized for one connection. Creating a new +# engine on every invocation left idle pooled connections open until the +# container died, which is how a burst of a few dozen concurrent invocations +# exhausted a db.t4g.micro's roughly 70 connection slots. +# +# Entries are keyed by the secret ARN, never by the connection URL: the URL +# carries the password, and keying on it would keep a stale pool alive for +# every rotation the container lives through. Each entry remembers the URL it +# was built from so a rotated credential disposes the old pool and builds one +# replacement, and whether the tracker tables have been created through it. +_DATABASE_ENGINES: dict[str, dict[str, Any]] = {} + + +def database_engine(cache_key: str, connection_string: str) -> Any: + """ + Return this container's SQLAlchemy engine for a tracker database. + + Parameters + ---------- + cache_key : str + Stable identifier for the database, normally the Secrets Manager + secret ARN. It must not contain credentials. + connection_string : str + PostgreSQL connection URL built from the current secret value. + + Returns + ------- + Any + A shared ``sqlalchemy.engine.Engine`` with a single pooled connection, + pre-ping enabled, and connections recycled after five minutes. When the + URL differs from the one the cached engine was built with, the old + engine is disposed and a replacement is created, so a rotated password + never leaves a stale pool behind. + """ + entry = _DATABASE_ENGINES.get(cache_key) + if entry is not None and entry["url"] == connection_string: + return entry["engine"] + + if entry is not None: + log.info( + { + "status": "INFO", + "message": "Tracker credentials changed; replacing the cached database engine", + "secret": cache_key, + } + ) + try: + entry["engine"].dispose() + except Exception as e: + log.warning(f"Could not dispose the previous database engine: {e}") + + engine = sqlalchemy_create_engine( + connection_string, + pool_size=1, + max_overflow=0, + pool_pre_ping=True, + pool_recycle=300, + ) + _DATABASE_ENGINES[cache_key] = { + "url": connection_string, + "engine": engine, + "tables_created": False, + } + return engine + + +def ensure_tracker_tables(cache_key: str, engine: Any) -> None: + """ + Create the tracker tables once per cached engine. + + Parameters + ---------- + cache_key : str + The key the engine is cached under. + engine : Any + The engine returned by :func:`database_engine` for that key. + """ + entry = _DATABASE_ENGINES.get(cache_key) + if entry is not None and entry["engine"] is engine and entry["tables_created"]: + return + create_tables(engine) + if entry is not None and entry["engine"] is engine: + entry["tables_created"] = True + def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]: """ @@ -425,14 +509,15 @@ def _track_file_metatracker( f"{secret['host']}:{secret['port']}/{secret['dbname']}" ) - # Initialize the database engine - database_engine = create_engine(connection_string) + # Reuse this container's engine instead of opening a new pool. + # The secret ARN is the cache key; the URL holds the password. + engine = database_engine(secret_arn, connection_string) - # Create tables if they do not exist - create_tables(database_engine) + # Create tables once per engine if they do not exist + ensure_tracker_tables(secret_arn, engine) # Set tracker to MetaTracker - meta_tracker = MetaTracker(database_engine, parse_science_filename) + meta_tracker = MetaTracker(engine, parse_science_filename) if meta_tracker: science_file_id, science_product_id = meta_tracker.track( diff --git a/lambda_function/tests/test_processor.py b/lambda_function/tests/test_processor.py index 6ea34f4..023dcff 100755 --- a/lambda_function/tests/test_processor.py +++ b/lambda_function/tests/test_processor.py @@ -5,6 +5,7 @@ import boto3 import pytest from moto import mock_aws as moto_mock_aws +from src.file_processor import file_processor as file_processor_module from src.file_processor.file_processor import FileProcessor, handle_event from swxsoc import log @@ -87,6 +88,7 @@ def test_file_calibrate( if n_expected > 0: assert calibrated_files[0] == expected_output + @pytest.mark.parametrize("use_mission", ["swxsoc_pipeline"], indirect=True) def test_file_calibrate_failure(use_mission): # Setup @@ -109,7 +111,7 @@ def test_handle_event(use_mission, s3_client, tmp_path, monkeypatch): filename = "REACH-TEST_20250904T000000_20250904T010000.csv" parent_dir = Path(__file__).parent / "test_data" - + # Set the absolute path using file_path as string os.environ["SDC_AWS_FILE_PATH"] = str(parent_dir / filename) @@ -134,3 +136,90 @@ def test_handle_event(use_mission, s3_client, tmp_path, monkeypatch): # Verify assert response["statusCode"] == 200 + + +class FakeEngine: + def __init__(self, url): + self.url = url + self.disposed = False + + def dispose(self): + self.disposed = True + + +def test_database_engine_is_shared_per_secret(monkeypatch): + created = [] + + def fake_create_engine(url, **kwargs): + created.append((url, kwargs)) + return FakeEngine(url) + + monkeypatch.setattr( + file_processor_module, "sqlalchemy_create_engine", fake_create_engine + ) + monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {}) + + first = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:p1@a/db" + ) + second = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:p1@a/db" + ) + other = file_processor_module.database_engine( + "arn:secret:b", "postgresql://u:p1@b/db" + ) + + assert first is second + assert other is not first + assert len(created) == 2 + assert created[0][1]["pool_size"] == 1 + assert created[0][1]["max_overflow"] == 0 + assert created[0][1]["pool_pre_ping"] is True + + +def test_rotated_password_replaces_the_cached_engine(monkeypatch): + monkeypatch.setattr( + file_processor_module, + "sqlalchemy_create_engine", + lambda url, **kw: FakeEngine(url), + ) + monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {}) + + before = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:old@a/db" + ) + after = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:new@a/db" + ) + again = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:new@a/db" + ) + + assert after is not before + assert before.disposed is True + assert again is after + assert list(file_processor_module._DATABASE_ENGINES) == ["arn:secret:a"] + + +def test_tracker_tables_are_created_once_per_engine(monkeypatch): + calls = [] + monkeypatch.setattr( + file_processor_module, + "sqlalchemy_create_engine", + lambda url, **kw: FakeEngine(url), + ) + monkeypatch.setattr(file_processor_module, "create_tables", calls.append) + monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {}) + + engine = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:old@a/db" + ) + file_processor_module.ensure_tracker_tables("arn:secret:a", engine) + file_processor_module.ensure_tracker_tables("arn:secret:a", engine) + assert calls == [engine] + + rotated = file_processor_module.database_engine( + "arn:secret:a", "postgresql://u:new@a/db" + ) + file_processor_module.ensure_tracker_tables("arn:secret:a", rotated) + assert calls == [engine, rotated]