diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index f94b678..bb4178e 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -85,16 +85,26 @@ jobs: fi echo "Promoting $config_type ($collection_id) to production" - response=$(python3 "$script_to_run" "$file" "production") + exit_code=0 + response=$(python3 "$script_to_run" "$file" "production" 2>&1) || exit_code=$? + + echo "Script output:" + echo "$response" + + if [ "$exit_code" -ne 0 ]; then + echo "[ERROR] $collection_id ($config_type) promotion script exited with code $exit_code" + failed=1 + continue + fi echo "Processed file: $file" - status_code=$(echo "$response" | jq -r '.statusCode') + status_code=$(echo "$response" | tail -n 1 | jq -r '.statusCode' 2>/dev/null || echo "unknown") echo "Status Code: $status_code" - if [ "$status_code" -eq 200 ] || [ "$status_code" -eq 201 ]; then - echo "$collection_id ($config_type) successfully promoted to Production ✅" - else - echo "$collection_id ($config_type) failed to promote to Production ❌" - failed=1 - fi + if [ "$status_code" = "200" ] || [ "$status_code" = "201" ]; then + echo "$collection_id ($config_type) successfully promoted to Production ✅" + else + echo "[ERROR] $collection_id ($config_type) failed to promote to Production ❌ (status: $status_code)" + failed=1 + fi done [ "${failed:-0}" -eq 0 ] || exit 1 \ No newline at end of file diff --git a/scripts/airflow_api.py b/scripts/airflow_api.py index 583f840..13cedb4 100644 --- a/scripts/airflow_api.py +++ b/scripts/airflow_api.py @@ -63,8 +63,8 @@ def trigger_dag_run( base_api_url: str, dag_id: str, conf: Dict[str, Any], - username: str, - password: str, + username: str = None, + password: str = None, api_version: str = None, ) -> Dict[str, Any]: """ diff --git a/scripts/promote_collection.py b/scripts/promote_collection.py index 80d6dd2..8408e94 100644 --- a/scripts/promote_collection.py +++ b/scripts/promote_collection.py @@ -33,8 +33,14 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str): password = os.getenv(password_env) api_version = os.getenv(api_version_env) or "2" - if not all([base_api_url, username, password]): - raise ValueError(f"Missing required environment variables for stage '{stage}' ") + # Airflow 3 authenticates with a JWT, so only require basic auth credentials for v2 + required = [base_api_url] + ([username, password] if api_version == "2" else []) + if not all(required): + raise ValueError( + f"Missing required environment variables for stage '{stage}' " + f"({api_url_env}, {username_env}, {password_env}; " + "username and password are only required for Airflow API v2)" + ) try: result = trigger_dag_run( diff --git a/scripts/promote_dataset.py b/scripts/promote_dataset.py index 1cd3e20..0779e50 100644 --- a/scripts/promote_dataset.py +++ b/scripts/promote_dataset.py @@ -28,10 +28,13 @@ def publish_to_staging(payload): password = os.getenv("STAGING_SM2A_ADMIN_PASSWORD") api_version_env = os.getenv("STAGING_AIRFLOW_API_VERSION") or "2" - if not all([base_api_url, username, password]): + # Airflow 3 authenticates with a JWT, so only require basic auth credentials for v2 + required = [base_api_url] + ([username, password] if api_version_env == "2" else []) + if not all(required): raise ValueError( "Missing required env vars STAGING_SM2A_API_URL, " - "STAGING_SM2A_ADMIN_USERNAME, STAGING_SM2A_ADMIN_PASSWORD" + "STAGING_SM2A_ADMIN_USERNAME, STAGING_SM2A_ADMIN_PASSWORD " + "(username and password are only required for Airflow API v2)" ) try: @@ -56,10 +59,13 @@ def promote_to_production(payload): password = os.getenv("SM2A_ADMIN_PASSWORD") api_version_env = os.getenv("PRODUCTION_AIRFLOW_API_VERSION") or "2" - if not all([base_api_url, username, password]): + # Airflow 3 authenticates with a JWT, so only require basic auth credentials for v2 + required = [base_api_url] + ([username, password] if api_version_env == "2" else []) + if not all(required): raise ValueError( "Missing required env vars SM2A_API_URL, " - "SM2A_ADMIN_USERNAME, SM2A_ADMIN_PASSWORD" + "SM2A_ADMIN_USERNAME, SM2A_ADMIN_PASSWORD " + "(username and password are only required for Airflow API v2)" ) payload["conf"].setdefault("transfer", False)