Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ This repository is to define the image to be used for the SWSOC file processing

The container will contain the latest release code as the production environment and the latest code on master as the development.

### **Database connections**:

The MetaTracker engine is created once per Lambda container with a single pooled connection, so a burst of invocations holds one connection per container instead of leaving pools open behind every event.

### **Testing Locally (Using own Test Data)**:
1. Build the lambda container image (from within the lambda_function folder) you'd like to test:

Expand Down
39 changes: 28 additions & 11 deletions buildspec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,26 +23,43 @@ phases:
- echo Login to Private ECR $REGION
- aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin 351967858401.dkr.ecr.$REGION.amazonaws.com
- echo ________________________________

- echo Create Date Tag
- export TAG=$(date +%Y-%m-%d-%H-%M-%S)
- ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' --output text)
- |
BASE_IMAGE="public.ecr.aws/w5r9l1c8/swsoc-docker-lambda-base:latest"
if git describe --tags --exact-match > /dev/null 2>&1; then
echo "This is a tag push event"
ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda"
CDK_ENVIRONMENT=PRODUCTION
VERSION=$(git describe --tags --exact-match)
elif [[ "${CDK_ENVIRONMENT}" == "PRODUCTION" ]]; then
echo "This is a production environment"
ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda"
CDK_ENVIRONMENT=PRODUCTION
VERSION=$(date -u +"%Y%m%d%H%M%S")
else
echo "This is a development environment"
ECR_REPO="$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/dev-sdc_aws_processing_lambda"
BASE_IMAGE="public.ecr.aws/w5r9l1c8/dev-swsoc-docker-lambda-base:latest"
CDK_ENVIRONMENT=DEVELOPMENT
VERSION=$(date -u +"%Y%m%d%H%M%S")
fi
- echo ________________________________

- echo Build Docker Image
- docker build -t 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest lambda_function/.
- docker build --build-arg BASE_IMAGE=$BASE_IMAGE -t $ECR_REPO:latest lambda_function/.

- echo Tagging Docker Image...
- docker tag 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:$TAG
- docker tag $ECR_REPO:latest $ECR_REPO:$VERSION

- echo Pushing the Docker image with Tags...
- docker push 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:latest
- docker push 351967858401.dkr.ecr.$REGION.amazonaws.com/sdc_aws_processing_lambda:$TAG
- docker push $ECR_REPO:latest
- docker push $ECR_REPO:$VERSION
- echo ________________________________

- echo Updating Deployment
- aws codebuild start-build --project-name build_sdc_aws_pipeline_architecture --region us-east-2 --environment-variables-override name=LAMBDA_PIPELINE,value=PROCESSING,type=PLAINTEXT name=TAG,value=$TAG,type=PLAINTEXT
- aws codebuild start-build --project-name build_sdc_aws_pipeline_architecture --region us-east-2 --environment-variables-override name=CDK_ENVIRONMENT,value=$CDK_ENVIRONMENT,type=PLAINTEXT
- echo ________________________________

post_build:
commands:
- echo Build Successful - Lambda Successfully Built and Pushed to ECR
- echo Build Successful - Lambda Successfully Built and Pushed to ECR
97 changes: 91 additions & 6 deletions lambda_function/src/file_processor/file_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,99 @@
import boto3
import psycopg2
import swxsoc
from sqlalchemy import create_engine as sqlalchemy_create_engine
from swxsoc import log
from swxsoc.db import create_engine
from swxsoc.db.tables import create_tables
from swxsoc.db.tracker import MetaTracker
from swxsoc.io.s3 import get_science_file, parse_file_key, push_science_file
from swxsoc.util.config import get_instrument_bucket, get_instrument_package
from swxsoc.util.util import parse_science_filename
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random

# One engine per Lambda container, sized for one connection. Creating a new
# engine on every invocation left idle pooled connections open until the
# container died, which is how a burst of a few dozen concurrent invocations
# exhausted a db.t4g.micro's roughly 70 connection slots.
#
# Entries are keyed by the secret ARN, never by the connection URL: the URL
# carries the password, and keying on it would keep a stale pool alive for
# every rotation the container lives through. Each entry remembers the URL it
# was built from so a rotated credential disposes the old pool and builds one
# replacement, and whether the tracker tables have been created through it.
_DATABASE_ENGINES: dict[str, dict[str, Any]] = {}


def database_engine(cache_key: str, connection_string: str) -> Any:
"""
Return this container's SQLAlchemy engine for a tracker database.

Parameters
----------
cache_key : str
Stable identifier for the database, normally the Secrets Manager
secret ARN. It must not contain credentials.
connection_string : str
PostgreSQL connection URL built from the current secret value.

Returns
-------
Any
A shared ``sqlalchemy.engine.Engine`` with a single pooled connection,
pre-ping enabled, and connections recycled after five minutes. When the
URL differs from the one the cached engine was built with, the old
engine is disposed and a replacement is created, so a rotated password
never leaves a stale pool behind.
"""
entry = _DATABASE_ENGINES.get(cache_key)
if entry is not None and entry["url"] == connection_string:
return entry["engine"]

if entry is not None:
log.info(
{
"status": "INFO",
"message": "Tracker credentials changed; replacing the cached database engine",
"secret": cache_key,
}
)
try:
entry["engine"].dispose()
except Exception as e:
log.warning(f"Could not dispose the previous database engine: {e}")

engine = sqlalchemy_create_engine(
connection_string,
pool_size=1,
max_overflow=0,
pool_pre_ping=True,
pool_recycle=300,
)
_DATABASE_ENGINES[cache_key] = {
"url": connection_string,
"engine": engine,
"tables_created": False,
}
return engine


def ensure_tracker_tables(cache_key: str, engine: Any) -> None:
"""
Create the tracker tables once per cached engine.

Parameters
----------
cache_key : str
The key the engine is cached under.
engine : Any
The engine returned by :func:`database_engine` for that key.
"""
entry = _DATABASE_ENGINES.get(cache_key)
if entry is not None and entry["engine"] is engine and entry["tables_created"]:
return
create_tables(engine)
if entry is not None and entry["engine"] is engine:
entry["tables_created"] = True


def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]:
"""
Expand Down Expand Up @@ -425,14 +509,15 @@ def _track_file_metatracker(
f"{secret['host']}:{secret['port']}/{secret['dbname']}"
)

# Initialize the database engine
database_engine = create_engine(connection_string)
# Reuse this container's engine instead of opening a new pool.
# The secret ARN is the cache key; the URL holds the password.
engine = database_engine(secret_arn, connection_string)

# Create tables if they do not exist
create_tables(database_engine)
# Create tables once per engine if they do not exist
ensure_tracker_tables(secret_arn, engine)

# Set tracker to MetaTracker
meta_tracker = MetaTracker(database_engine, parse_science_filename)
meta_tracker = MetaTracker(engine, parse_science_filename)

if meta_tracker:
science_file_id, science_product_id = meta_tracker.track(
Expand Down
91 changes: 90 additions & 1 deletion lambda_function/tests/test_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import boto3
import pytest
from moto import mock_aws as moto_mock_aws
from src.file_processor import file_processor as file_processor_module
from src.file_processor.file_processor import FileProcessor, handle_event
from swxsoc import log

Expand Down Expand Up @@ -87,6 +88,7 @@ def test_file_calibrate(
if n_expected > 0:
assert calibrated_files[0] == expected_output


@pytest.mark.parametrize("use_mission", ["swxsoc_pipeline"], indirect=True)
def test_file_calibrate_failure(use_mission):
# Setup
Expand All @@ -109,7 +111,7 @@ def test_handle_event(use_mission, s3_client, tmp_path, monkeypatch):

filename = "REACH-TEST_20250904T000000_20250904T010000.csv"
parent_dir = Path(__file__).parent / "test_data"

# Set the absolute path using file_path as string
os.environ["SDC_AWS_FILE_PATH"] = str(parent_dir / filename)

Expand All @@ -134,3 +136,90 @@ def test_handle_event(use_mission, s3_client, tmp_path, monkeypatch):

# Verify
assert response["statusCode"] == 200


class FakeEngine:
def __init__(self, url):
self.url = url
self.disposed = False

def dispose(self):
self.disposed = True


def test_database_engine_is_shared_per_secret(monkeypatch):
created = []

def fake_create_engine(url, **kwargs):
created.append((url, kwargs))
return FakeEngine(url)

monkeypatch.setattr(
file_processor_module, "sqlalchemy_create_engine", fake_create_engine
)
monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {})

first = file_processor_module.database_engine(
"arn:secret:a", "postgresql://u:p1@a/db"
)
second = file_processor_module.database_engine(
"arn:secret:a", "postgresql://u:p1@a/db"
)
other = file_processor_module.database_engine(
"arn:secret:b", "postgresql://u:p1@b/db"
)

assert first is second
assert other is not first
assert len(created) == 2
assert created[0][1]["pool_size"] == 1
assert created[0][1]["max_overflow"] == 0
assert created[0][1]["pool_pre_ping"] is True


def test_rotated_password_replaces_the_cached_engine(monkeypatch):
monkeypatch.setattr(
file_processor_module,
"sqlalchemy_create_engine",
lambda url, **kw: FakeEngine(url),
)
monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {})

before = file_processor_module.database_engine(
"arn:secret:a", "postgresql://u:old@a/db"
)
after = file_processor_module.database_engine(
"arn:secret:a", "postgresql://u:new@a/db"
)
again = file_processor_module.database_engine(
"arn:secret:a", "postgresql://u:new@a/db"
)

assert after is not before
assert before.disposed is True
assert again is after
assert list(file_processor_module._DATABASE_ENGINES) == ["arn:secret:a"]


def test_tracker_tables_are_created_once_per_engine(monkeypatch):
calls = []
monkeypatch.setattr(
file_processor_module,
"sqlalchemy_create_engine",
lambda url, **kw: FakeEngine(url),
)
monkeypatch.setattr(file_processor_module, "create_tables", calls.append)
monkeypatch.setattr(file_processor_module, "_DATABASE_ENGINES", {})

engine = file_processor_module.database_engine(
"arn:secret:a", "postgresql://u:old@a/db"
)
file_processor_module.ensure_tracker_tables("arn:secret:a", engine)
file_processor_module.ensure_tracker_tables("arn:secret:a", engine)
assert calls == [engine]

rotated = file_processor_module.database_engine(
"arn:secret:a", "postgresql://u:new@a/db"
)
file_processor_module.ensure_tracker_tables("arn:secret:a", rotated)
assert calls == [engine, rotated]
Loading