fix: update promotion scripts to not require username and password for airflow v3

.github/workflows/promote.yml

@@ -85,16 +85,26 @@ jobs:
             fi
 
             echo "Promoting $config_type ($collection_id) to production"
-            response=$(python3 "$script_to_run" "$file" "production")
+            exit_code=0
+            response=$(python3 "$script_to_run" "$file" "production" 2>&1) || exit_code=$?
+
+            echo "Script output:"
+            echo "$response"
+
+            if [ "$exit_code" -ne 0 ]; then
+              echo "[ERROR] $collection_id ($config_type) promotion script exited with code $exit_code"
+              failed=1
+              continue
+            fi
 
             echo "Processed file: $file"
-            status_code=$(echo "$response" | jq -r '.statusCode')
+            status_code=$(echo "$response" | tail -n 1 | jq -r '.statusCode' 2>/dev/null || echo "unknown")
             echo "Status Code: $status_code"
-             if [ "$status_code" -eq 200 ] || [ "$status_code" -eq 201 ]; then
-                echo "$collection_id ($config_type) successfully promoted to Production ✅"
-              else
-                echo "$collection_id ($config_type) failed to promote to Production ❌"
-                failed=1
-              fi
+            if [ "$status_code" = "200" ] || [ "$status_code" = "201" ]; then
+              echo "$collection_id ($config_type) successfully promoted to Production ✅"
+            else
+              echo "[ERROR] $collection_id ($config_type) failed to promote to Production ❌ (status: $status_code)"
+              failed=1
+            fi
           done
           [ "${failed:-0}" -eq 0 ] || exit 1

scripts/airflow_api.py

@@ -63,8 +63,8 @@ def trigger_dag_run(
     base_api_url: str,
     dag_id: str,
     conf: Dict[str, Any],
-    username: str,
-    password: str,
+    username: str = None,
+    password: str = None,
     api_version: str = None,
 ) -> Dict[str, Any]:
     """

scripts/promote_collection.py

@@ -33,8 +33,14 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str):
     password = os.getenv(password_env)
     api_version = os.getenv(api_version_env) or "2"
 
-    if not all([base_api_url, username, password]):
-        raise ValueError(f"Missing required environment variables for stage '{stage}' ")
+    # Airflow 3 authenticates with a JWT, so only require basic auth credentials for v2
+    required = [base_api_url] + ([username, password] if api_version == "2" else [])
+    if not all(required):
+        raise ValueError(
+            f"Missing required environment variables for stage '{stage}' "
+            f"({api_url_env}, {username_env}, {password_env}; "
+            "username and password are only required for Airflow API v2)"
+        )
 
     try:
         result = trigger_dag_run(

scripts/promote_dataset.py

@@ -28,10 +28,13 @@ def publish_to_staging(payload):
     password = os.getenv("STAGING_SM2A_ADMIN_PASSWORD")
     api_version_env = os.getenv("STAGING_AIRFLOW_API_VERSION") or "2"
 
-    if not all([base_api_url, username, password]):
+    # Airflow 3 authenticates with a JWT, so only require basic auth credentials for v2
+    required = [base_api_url] + ([username, password] if api_version_env == "2" else [])
+    if not all(required):
         raise ValueError(
             "Missing required env vars STAGING_SM2A_API_URL, "
-            "STAGING_SM2A_ADMIN_USERNAME, STAGING_SM2A_ADMIN_PASSWORD"
+            "STAGING_SM2A_ADMIN_USERNAME, STAGING_SM2A_ADMIN_PASSWORD "
+            "(username and password are only required for Airflow API v2)"
         )
 
     try:
@@ -56,10 +59,13 @@ def promote_to_production(payload):
     password = os.getenv("SM2A_ADMIN_PASSWORD")
     api_version_env = os.getenv("PRODUCTION_AIRFLOW_API_VERSION") or "2"
 
-    if not all([base_api_url, username, password]):
+    # Airflow 3 authenticates with a JWT, so only require basic auth credentials for v2
+    required = [base_api_url] + ([username, password] if api_version_env == "2" else [])
+    if not all(required):
         raise ValueError(
             "Missing required env vars SM2A_API_URL, "
-            "SM2A_ADMIN_USERNAME, SM2A_ADMIN_PASSWORD"
+            "SM2A_ADMIN_USERNAME, SM2A_ADMIN_PASSWORD "
+            "(username and password are only required for Airflow API v2)"
         )
 
     payload["conf"].setdefault("transfer", False)