From f8c363fee882b34ec27ba3115cf7b49e023c7869 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 13:22:09 -0400 Subject: [PATCH 1/5] [Test] proxy infra: set common curl retry/timing options (--retry 3 --connect-timeout 5 --max-time 15) to explicitly fail on unresponsive endpoints, consistently for all the tested endpoints. --- cloudformation/proxy/proxy.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index 3713221806..04c6030a20 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -347,9 +347,14 @@ Resources: set -o pipefail exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + # Common curl options, used for every request below: bounded retries for transient + # glitches plus connect/total time caps so no single hung endpoint can block the script + # past the wait-condition budget. + CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15" + # Signal the success/failure to the wait condition. function signal() { - curl -X PUT -H "Content-Type:" \ + curl $CURL_OPTS -X PUT -H "Content-Type:" \ --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"Proxy\",\"Data\":\"$1\"}" \ "${ProxyReadyWaitConditionHandle}" } @@ -469,7 +474,7 @@ Resources: # the mirrorlist here and allowlist every host it returns. for RELEASEVER in 8 9; do for ARCH in x86_64 aarch64; do - curl -s --retry 5 "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \ + curl -s $CURL_OPTS "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \ | awk -F/ '/^https?:/ {print $3}' done done | sort -u | while read -r EPEL_HOST; do @@ -573,6 +578,11 @@ Resources: set -o pipefail exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + # Common curl options, used for every request below: bounded retries for transient + # glitches plus connect/total time caps so no single hung endpoint can block the script + # past the wait-condition budget. + CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15" + # Resolve this instance's ID via IMDSv2, for troubleshooting. Never fails: echoes # "unknown" if the metadata lookup does not succeed. function get_instance_id() { @@ -583,7 +593,7 @@ Resources: # Signal the success/failure to the wait condition. function signal() { - curl -X PUT -H "Content-Type:" \ + curl $CURL_OPTS -X PUT -H "Content-Type:" \ --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"ProxyClient\",\"Data\":\"$1\"}" \ "${ProxyVerificationWaitConditionHandle}" } @@ -623,7 +633,7 @@ Resources: if [ "$BUILD_IMAGE_PROXY" = "true" ]; then echo "==> Testing HTTPS proxy (same as build instance uses via https_proxy env var)" - https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1 + https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v $CURL_OPTS -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1 echo "==> HTTPS transparent proxy test passed" # Verify the proxy allowlisted every EPEL mirror the mirrorlist returns. @@ -632,12 +642,12 @@ Resources: PROXY="http://${ProxyPrivateIp}:${ProxyPort}" DENIED="" for RELEASEVER in 8 9; do - MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s --retry 5 \ + MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s $CURL_OPTS \ "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=x86_64") for URL in $MIRRORS; do case "$URL" in http*) ;; *) continue ;; esac BASE=$(echo "$URL" | sed 's#/*$#/#') - RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v --retry 2 -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true) + RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true) if echo "$RESP" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'; then echo "==> DENIED by proxy (EPEL $RELEASEVER mirror not allowlisted): $URL" DENIED="$DENIED $URL" From b03a0ed4fc78f99e71f70f38334280b816345cc5 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 13:32:59 -0400 Subject: [PATCH 2/5] [Test] proxy infra: make the proxy client verification resilient against network glitches. The client now verifies only what it was supposed to verify: that every expected endpoint is reachable through the proxy. IF the endpoint is temporarily unresponsive, that does not matter. --- cloudformation/proxy/proxy.yaml | 40 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index 04c6030a20..910557e247 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -583,6 +583,11 @@ Resources: # past the wait-condition budget. CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15" + # True when $1 (curl -v output) shows the proxy blocked the request. + function proxy_denied() { + echo "$1" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied' + } + # Resolve this instance's ID via IMDSv2, for troubleshooting. Never fails: echoes # "unknown" if the metadata lookup does not succeed. function get_instance_id() { @@ -632,35 +637,34 @@ Resources: apt-get $APT_RETRY update -y if [ "$BUILD_IMAGE_PROXY" = "true" ]; then - echo "==> Testing HTTPS proxy (same as build instance uses via https_proxy env var)" - https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v $CURL_OPTS -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1 - echo "==> HTTPS transparent proxy test passed" - - # Verify the proxy allowlisted every EPEL mirror the mirrorlist returns. - # Fail only on a proxy denial (allowlist gap), not on a mirror being down. - echo "==> Validating EPEL mirror allowlisting through the proxy" PROXY="http://${ProxyPrivateIp}:${ProxyPort}" - DENIED="" + + # Endpoints the build instance reaches through the proxy + ENDPOINTS="https://api.snapcraft.io/v2/snaps/info/core" for RELEASEVER in 8 9; do MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s $CURL_OPTS \ "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=x86_64") for URL in $MIRRORS; do case "$URL" in http*) ;; *) continue ;; esac - BASE=$(echo "$URL" | sed 's#/*$#/#') - RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true) - if echo "$RESP" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'; then - echo "==> DENIED by proxy (EPEL $RELEASEVER mirror not allowlisted): $URL" - DENIED="$DENIED $URL" - else - echo "==> Allowed by proxy (EPEL $RELEASEVER): $URL" - fi + ENDPOINTS="$ENDPOINTS $(echo "$URL" | sed 's#/*$#/#')repodata/repomd.xml" done done + + # Fail only if the proxy blocks an endpoint, not on a glitch/unresponsive origin. + DENIED="" + for URL in $ENDPOINTS; do + RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$URL" 2>&1 || true) + if proxy_denied "$RESP"; then + echo "==> DENIED by proxy: $URL" + DENIED="$DENIED $URL" + else + echo "==> Allowed by proxy: $URL" + fi + done if [ -n "$DENIED" ]; then - echo "==> ERROR: the proxy denied EPEL mirrors that should have been allowlisted:$DENIED" + echo "==> ERROR: the proxy blocked endpoints that should have been allowlisted:$DENIED" exit 1 fi - echo "==> EPEL mirror allowlisting validated for all mirrors" echo "==> Signaling success" signal SUCCESS "Proxy verification passed" From 27f88c6587bb859d3608067e95354b2fcf2e1476 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 14:56:39 -0400 Subject: [PATCH 3/5] [Test] Pin RHEL9 parent image used for build-image tests to version 9.8. This pinning is needed because previous versions require the paid Extended Update Support (EUS) repos to be configured to install required packages, such as kernel-devel. --- tests/integration-tests/tests/common/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration-tests/tests/common/utils.py b/tests/integration-tests/tests/common/utils.py index 394d01f60a..8bf4f2ce3b 100644 --- a/tests/integration-tests/tests/common/utils.py +++ b/tests/integration-tests/tests/common/utils.py @@ -71,7 +71,9 @@ }, # TODO add china and govcloud accounts "rhel8.9": {"name": "RHEL-8.9*_HVM-*", "owners": RHEL_OWNERS}, "rocky8.9": {"name": "Rocky-8-EC2-Base-8.9*", "owners": ["792107900819"]}, # TODO add china and govcloud accounts - "rhel9": {"name": "RHEL-9.*_HVM*", "owners": RHEL_OWNERS}, + # Pin to the latest RHEL 9.8 as previous minor requires paid Extended Update Support (EUS) repo + # to install packages we need, such as kernel packages. + "rhel9": {"name": "RHEL-9.8*_HVM*", "owners": RHEL_OWNERS}, "rocky9": {"name": "Rocky-9-EC2-Base-9.*", "owners": ["792107900819"]}, # TODO add china and govcloud accounts } From b8d3a661734e96ba9910fa6e9b3fa24bdd0eef7a Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 15:04:21 -0400 Subject: [PATCH 4/5] [Test] Make dynamic ODCR framework honor the AZ allowlisting. This fix is needed the shared test VPC is only built in the allowlisted AZs, so a reservation placed outside them would pin the test to an AZ with no matching subnet. --- .../framework/tests_configuration/config_renderer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration-tests/framework/tests_configuration/config_renderer.py b/tests/integration-tests/framework/tests_configuration/config_renderer.py index 74afe72236..460621273b 100644 --- a/tests/integration-tests/framework/tests_configuration/config_renderer.py +++ b/tests/integration-tests/framework/tests_configuration/config_renderer.py @@ -20,6 +20,7 @@ import boto3 import yaml from botocore.exceptions import ClientError +from conftest_networking import AVAILABLE_AVAILABILITY_ZONE from jinja2 import FileSystemLoader, meta from jinja2.sandbox import SandboxedEnvironment from utils import InstanceTypesData @@ -642,10 +643,17 @@ def _create_capacity_reservations(az_for_cr, regions, specs, var): # noqa C901 for region in regions: try: ec2_client = boto3.client("ec2", region_name=region) + # Honor the AZ allowlist: the shared test VPC is only built in the allowlisted AZs, so a + # reservation placed outside them would pin the test to an AZ with no matching subnet. + # No entry for the region means all its available AZs are eligible; an entry restricts + # placement to exactly the listed AZs. + allowlisted_az_ids = AVAILABLE_AVAILABILITY_ZONE.get(region) for az in ec2_client.describe_availability_zones()["AvailabilityZones"]: if az["ZoneType"] != "availability-zone": continue zone_id = az["ZoneId"] + if allowlisted_az_ids is not None and zone_id not in allowlisted_az_ids: + continue created_capacity_reservation_ids = [] success = True for instance_type, os_platform, count, end_date, enable_placement_group in specs: From e2901393b07b2ed0f3f74b07d7b97172c4c00e80 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 15:51:59 -0400 Subject: [PATCH 5/5] [Test] build_image_no_internet: make the build-image config used by the test honor the flag to enable/disable Lustre installation. As a result, we disable Lustre installation on ubuntu, the same way we do for test_build_image. --- tests/integration-tests/tests/createami/test_createami.py | 1 + .../test_build_image_no_internet/image.config.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/integration-tests/tests/createami/test_createami.py b/tests/integration-tests/tests/createami/test_createami.py index 9b6a28950e..69e17e892f 100644 --- a/tests/integration-tests/tests/createami/test_createami.py +++ b/tests/integration-tests/tests/createami/test_createami.py @@ -129,6 +129,7 @@ def test_build_image_no_internet( node_package=s3_artifacts["node_package"], install_http_proxy_address=install_http_proxy_address, enable_nvidia=str(enable_nvidia).lower(), + enable_lustre_client=str(feature_flags["enable_lustre_client"]).lower(), ) image = images_factory(image_id, image_config, region) diff --git a/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml b/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml index 6e8f43468f..ec7e64820c 100644 --- a/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml +++ b/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml @@ -13,6 +13,8 @@ Build: AdditionalIamPolicies: - Policy: arn:{{ partition }}:iam::aws:policy/AmazonS3ReadOnlyAccess Installation: + LustreClient: + Enabled: {{ enable_lustre_client }} NvidiaSoftware: Enabled: {{ enable_nvidia }}