[Test] Test stabilization fixes for 3.16.1: patching, build-image no-internet, multi_az_fsx and AZ allowlisting

cloudformation/proxy/proxy.yaml

@@ -347,9 +347,14 @@ Resources:
             set -o pipefail
             exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 
+            # Common curl options, used for every request below: bounded retries for transient
+            # glitches plus connect/total time caps so no single hung endpoint can block the script
+            # past the wait-condition budget.
+            CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15"
+
             # Signal the success/failure to the wait condition.
             function signal() {
-              curl -X PUT -H "Content-Type:" \
+              curl $CURL_OPTS -X PUT -H "Content-Type:" \
                 --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"Proxy\",\"Data\":\"$1\"}" \
                 "${ProxyReadyWaitConditionHandle}"
             }
@@ -469,7 +474,7 @@ Resources:
               # the mirrorlist here and allowlist every host it returns.
               for RELEASEVER in 8 9; do
                 for ARCH in x86_64 aarch64; do
-                  curl -s --retry 5 "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \
+                  curl -s $CURL_OPTS "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \
                     | awk -F/ '/^https?:/ {print $3}'
                 done
               done | sort -u | while read -r EPEL_HOST; do
@@ -573,6 +578,16 @@ Resources:
             set -o pipefail
             exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 
+            # Common curl options, used for every request below: bounded retries for transient
+            # glitches plus connect/total time caps so no single hung endpoint can block the script
+            # past the wait-condition budget.
+            CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15"
+
+            # True when $1 (curl -v output) shows the proxy blocked the request.
+            function proxy_denied() {
+              echo "$1" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'
+            }
+
             # Resolve this instance's ID via IMDSv2, for troubleshooting. Never fails: echoes
             # "unknown" if the metadata lookup does not succeed.
             function get_instance_id() {
@@ -583,7 +598,7 @@ Resources:
 
             # Signal the success/failure to the wait condition.
             function signal() {
-              curl -X PUT -H "Content-Type:" \
+              curl $CURL_OPTS -X PUT -H "Content-Type:" \
                 --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"ProxyClient\",\"Data\":\"$1\"}" \
                 "${ProxyVerificationWaitConditionHandle}"
             }
@@ -622,35 +637,34 @@ Resources:
             apt-get $APT_RETRY update -y
 
             if [ "$BUILD_IMAGE_PROXY" = "true" ]; then
-              echo "==> Testing HTTPS proxy (same as build instance uses via https_proxy env var)"
-              https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1
-              echo "==> HTTPS transparent proxy test passed"
-
-              # Verify the proxy allowlisted every EPEL mirror the mirrorlist returns.
-              # Fail only on a proxy denial (allowlist gap), not on a mirror being down.
-              echo "==> Validating EPEL mirror allowlisting through the proxy"
               PROXY="http://${ProxyPrivateIp}:${ProxyPort}"
-              DENIED=""
+
+              # Endpoints the build instance reaches through the proxy
+              ENDPOINTS="https://api.snapcraft.io/v2/snaps/info/core"
               for RELEASEVER in 8 9; do
-                MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s --retry 5 \
+                MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s $CURL_OPTS \
                   "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=x86_64")
                 for URL in $MIRRORS; do
                   case "$URL" in http*) ;; *) continue ;; esac
-                  BASE=$(echo "$URL" | sed 's#/*$#/#')
-                  RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v --retry 2 -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true)
-                  if echo "$RESP" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'; then
-                    echo "==> DENIED by proxy (EPEL $RELEASEVER mirror not allowlisted): $URL"
-                    DENIED="$DENIED $URL"
-                  else
-                    echo "==> Allowed by proxy (EPEL $RELEASEVER): $URL"
-                  fi
+                  ENDPOINTS="$ENDPOINTS $(echo "$URL" | sed 's#/*$#/#')repodata/repomd.xml"
                 done
               done
+
+              # Fail only if the proxy blocks an endpoint, not on a glitch/unresponsive origin.
+              DENIED=""
+              for URL in $ENDPOINTS; do
+                RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$URL" 2>&1 || true)
+                if proxy_denied "$RESP"; then
+                  echo "==> DENIED by proxy: $URL"
+                  DENIED="$DENIED $URL"
+                else
+                  echo "==> Allowed by proxy: $URL"
+                fi
+              done
               if [ -n "$DENIED" ]; then
-                echo "==> ERROR: the proxy denied EPEL mirrors that should have been allowlisted:$DENIED"
+                echo "==> ERROR: the proxy blocked endpoints that should have been allowlisted:$DENIED"
                 exit 1
               fi
-              echo "==> EPEL mirror allowlisting validated for all mirrors"
 
               echo "==> Signaling success"
               signal SUCCESS "Proxy verification passed"

tests/integration-tests/framework/tests_configuration/config_renderer.py

@@ -20,6 +20,7 @@
 import boto3
 import yaml
 from botocore.exceptions import ClientError
+from conftest_networking import AVAILABLE_AVAILABILITY_ZONE
 from jinja2 import FileSystemLoader, meta
 from jinja2.sandbox import SandboxedEnvironment
 from utils import InstanceTypesData
@@ -642,10 +643,17 @@ def _create_capacity_reservations(az_for_cr, regions, specs, var):  # noqa C901
     for region in regions:
         try:
             ec2_client = boto3.client("ec2", region_name=region)
+            # Honor the AZ allowlist: the shared test VPC is only built in the allowlisted AZs, so a
+            # reservation placed outside them would pin the test to an AZ with no matching subnet.
+            # No entry for the region means all its available AZs are eligible; an entry restricts
+            # placement to exactly the listed AZs.
+            allowlisted_az_ids = AVAILABLE_AVAILABILITY_ZONE.get(region)
             for az in ec2_client.describe_availability_zones()["AvailabilityZones"]:
                 if az["ZoneType"] != "availability-zone":
                     continue
                 zone_id = az["ZoneId"]
+                if allowlisted_az_ids is not None and zone_id not in allowlisted_az_ids:
+                    continue
                 created_capacity_reservation_ids = []
                 success = True
                 for instance_type, os_platform, count, end_date, enable_placement_group in specs:

tests/integration-tests/tests/common/utils.py

@@ -71,7 +71,9 @@
     },  # TODO add china and govcloud accounts
     "rhel8.9": {"name": "RHEL-8.9*_HVM-*", "owners": RHEL_OWNERS},
     "rocky8.9": {"name": "Rocky-8-EC2-Base-8.9*", "owners": ["792107900819"]},  # TODO add china and govcloud accounts
-    "rhel9": {"name": "RHEL-9.*_HVM*", "owners": RHEL_OWNERS},
+    # Pin to the latest RHEL 9.8 as previous minor requires paid Extended Update Support (EUS) repo
+    # to install packages we need, such as kernel packages.
+    "rhel9": {"name": "RHEL-9.8*_HVM*", "owners": RHEL_OWNERS},
     "rocky9": {"name": "Rocky-9-EC2-Base-9.*", "owners": ["792107900819"]},  # TODO add china and govcloud accounts
 }
 

tests/integration-tests/tests/createami/test_createami.py

@@ -129,6 +129,7 @@ def test_build_image_no_internet(
         node_package=s3_artifacts["node_package"],
         install_http_proxy_address=install_http_proxy_address,
         enable_nvidia=str(enable_nvidia).lower(),
+        enable_lustre_client=str(feature_flags["enable_lustre_client"]).lower(),
     )
 
     image = images_factory(image_id, image_config, region)

tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml

@@ -13,6 +13,8 @@ Build:
         AdditionalIamPolicies:
             - Policy: arn:{{ partition }}:iam::aws:policy/AmazonS3ReadOnlyAccess
     Installation:
+        LustreClient:
+            Enabled: {{ enable_lustre_client }}
         NvidiaSoftware:
             Enabled: {{ enable_nvidia }}