[Test] Stabilize test_multiple_efs, test_queue_parameter_update, test_proxy, test_build_no_internet, test_cluster_with_gpu_health_checks

cloudformation/proxy/proxy.yaml

@@ -623,11 +623,11 @@ Resources:
   ProxyVerificationWaitCondition:
     Type: AWS::CloudFormation::WaitCondition
     DependsOn:
-      - Proxy
+      - ProxyClient
     Properties:
       Count: 1
       Handle: !Ref ProxyVerificationWaitConditionHandle
-      Timeout: 600
+      Timeout: 900
 
 
 Outputs:

tests/integration-tests/configs/develop.yaml

@@ -322,7 +322,7 @@ test-suites:
   health_checks:
     test_gpu_health_checks.py::test_cluster_with_gpu_health_checks:
       dimensions:
-        - regions: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0_CAPACITY_RESERVATION_3_INSTANCES_1_HOURS_NOPG_OS_X86_5 }}]
+        - regions: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0_CAPACITY_RESERVATION_4_INSTANCES_1_HOURS_NOPG_OS_X86_5__c5_xlarge_CAPACITY_RESERVATION_5_INSTANCES_1_HOURS_NOPG_OS_X86_5 }}]
           instances: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0 }}]
           oss: [{{ OS_X86_5 }}]
           schedulers: ["slurm"]

tests/integration-tests/tests/common/schedulers_common.py

@@ -496,6 +496,15 @@ def get_node_addr(self, node_name):
         node_addr = re.search(r"NodeAddr=(.*) NodeHostName", result).group(1)
         return node_addr
 
+    def get_job_instance_id(self, job_id):
+        """Return the id of the EC2 instance the job is running on, as reported by Slurm."""
+        node_name = self._remote_command_executor.run_remote_command(
+            f'scontrol show jobs {job_id} --json | jq -r ".jobs[0].batch_host"'
+        ).stdout.strip()
+        return self._remote_command_executor.run_remote_command(
+            f'scontrol show nodes {node_name} --json | jq -r ".nodes[0].instance_id"'
+        ).stdout.strip()
+
     def submit_command_and_assert_job_accepted(self, submit_command_args):
         """Submit a command and assert the job is accepted by scheduler."""
         result = self.submit_command(**submit_command_args)
@@ -520,6 +529,12 @@ def wait_job_running(self, job_id):
         result = self._remote_command_executor.run_remote_command("scontrol show jobs -o {0}".format(job_id))
         assert_that(result.stdout).contains("JobState=RUNNING")
 
+    @retry(wait_fixed=seconds(10), stop_max_delay=minutes(13))
+    def wait_job_requeued(self, job_id, times=1):
+        """Wait till the job has been requeued at least `times` times (Restarts>=times)."""
+        restarts = self.get_job_info(job_id, field="Restarts")
+        assert_that(int(restarts)).is_greater_than_or_equal_to(times)
+
     def get_node_info(self, nodename):
         """Get node info."""
         return self._remote_command_executor.run_remote_command("scontrol show nodes {0}".format(nodename)).stdout

tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml

@@ -93,9 +93,7 @@ Scheduling:
         - InstanceType: {{ non_gpu_instance }}
     Networking:
       SubnetIds:
-        {% for private_subnet_id in private_subnet_ids %}
         - {{ private_subnet_id }}
-        {% endfor %}
   - Name: queue-2
     ComputeResources:
     - Name: compute-resource-1
@@ -112,7 +110,5 @@ Scheduling:
           Enabled: true
     Networking:
       SubnetIds:
-        {% for private_subnet_id in private_subnet_ids %}
         - {{ private_subnet_id }}
-        {% endfor %}
 {% endif %}

tests/integration-tests/tests/patching/test_patching.py

@@ -53,39 +53,41 @@
 # may or may not have run by the time we snapshot). We force-load these after patching so that
 # the before/after comparison does not flag them as missing.
 # Maintained per OS and per node type. Keep module names alphabetically sorted.
+# Modules tolerated on the head node across all OSes.
+COMMON_HEAD_NODE_LAZY_MODULES = ["crc32_generic", "tls"]
 LAZY_KERNEL_MODULES = {
     "alinux2023": {
-        HEAD_NODE: ["tls"],
+        HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES,
         COMPUTE_NODE: ["tls"],
         LOGIN_NODE: ["tls"],
     },
     "rhel8": {
-        HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"],
+        HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"],
         COMPUTE_NODE: ["tls"],
         LOGIN_NODE: ["tls"],
     },
     "rhel9": {
-        HEAD_NODE: ["tls"],
+        HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES,
         COMPUTE_NODE: ["tls"],
         LOGIN_NODE: ["tls"],
     },
     "rocky8": {
-        HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"],
+        HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"],
         COMPUTE_NODE: ["tls"],
         LOGIN_NODE: ["tls"],
     },
     "rocky9": {
-        HEAD_NODE: ["tls"],
+        HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES,
         COMPUTE_NODE: ["tls"],
         LOGIN_NODE: ["tls"],
     },
     "ubuntu2204": {
-        HEAD_NODE: ["tls"],
+        HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES,
         COMPUTE_NODE: ["tls"],
         LOGIN_NODE: ["tls"],
     },
     "ubuntu2404": {
-        HEAD_NODE: ["tls"],
+        HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES,
         COMPUTE_NODE: ["tls"],
         LOGIN_NODE: ["tls"],
     },

tests/integration-tests/tests/storage/test_efs.py

@@ -17,6 +17,8 @@
 from assertpy import assert_that
 from cfn_stacks_factory import CfnVpcStack
 from remote_command_executor import RemoteCommandExecutor
+from retrying import retry
+from time_utils import minutes, seconds
 from utils import get_arn_partition, get_compute_nodes_instance_ips
 
 from tests.common.utils import get_sts_endpoint, reboot_head_node
@@ -350,12 +352,22 @@ def _check_efs_correctly_mounted_and_shared(
     all_mount_dirs, remote_command_executor, scheduler_commands, iam_authorizations, encryption_in_transits
 ):
     for i, mount_dir in enumerate(all_mount_dirs):
-        test_efs_correctly_mounted(
-            remote_command_executor,
-            mount_dir,
-            encryption_in_transits[i],
-            iam_authorizations[i],
-        )
+        if iam_authorizations[i]:
+            # An EFS with IAM authorization can be transiently slower to mount because the mount depends on
+            # IAM policy evaluation. As such, we retry the assertion to prevent flaky test failures.
+            retry(wait_fixed=seconds(10), stop_max_delay=minutes(3))(test_efs_correctly_mounted)(
+                remote_command_executor,
+                mount_dir,
+                encryption_in_transits[i],
+                iam_authorizations[i],
+            )
+        else:
+            test_efs_correctly_mounted(
+                remote_command_executor,
+                mount_dir,
+                encryption_in_transits[i],
+                iam_authorizations[i],
+            )
         _test_efs_correctly_shared(remote_command_executor, mount_dir, scheduler_commands)
 
 

tests/integration-tests/tests/update/test_update.py

@@ -42,7 +42,12 @@
     wait_for_computefleet_changed,
 )
 
-from tests.common.assertions import assert_instance_config_version_on_ddb, assert_lines_in_logs, assert_no_msg_in_logs
+from tests.common.assertions import (
+    assert_instance_config_version_on_ddb,
+    assert_lines_in_logs,
+    assert_no_msg_in_logs,
+    wait_instance_replaced_or_terminating,
+)
 from tests.common.hit_common import (
     assert_compute_node_states,
     assert_initial_conditions,
@@ -888,6 +893,10 @@ def _test_update_queue_strategy_with_running_job(
     scheduler_commands.wait_job_running(queue2_job_id)
     logging.info(f"Job {queue2_job_id} is running on queue2")
 
+    # Ask Slurm exactly which instance the queue2 job is running on, so that after a TERMINATE update
+    # we can assert that exact instance was terminated (the job's node was replaced).
+    queue2_instance_id = scheduler_commands.get_job_instance_id(queue2_job_id)
+
     logging.info(f"Updating cluster with strategy {queue_update_strategy} with running jobs")
     updated_config_file = pcluster_config_reader(
         config_file="pcluster.config.update_with_running_job.yaml",
@@ -913,8 +922,14 @@ def _test_update_queue_strategy_with_running_job(
     queue1_nodes = scheduler_commands.get_compute_nodes("queue1")
     assert_compute_node_states(scheduler_commands, queue1_nodes, expected_states=["mixed", "allocated"])
     if queue_update_strategy == "TERMINATE":
-        time.sleep(10)
-        scheduler_commands.assert_job_state(queue2_job_id, "CONFIGURING")
+        # TERMINATE forcibly terminates queue2's compute nodes and requeues the running job onto
+        # freshly launched replacement nodes. Assert the deterministic outcome (the job was requeued
+        # and is running again on the replaced instances) rather than snapshotting the transient
+        # CONFIGURING state, which can close before we sample it when replacement nodes boot quickly
+        # (the whole swap can finish inside the update-cluster --wait window).
+        scheduler_commands.wait_job_requeued(queue2_job_id)
+        wait_instance_replaced_or_terminating(queue2_instance_id, region)
+        scheduler_commands.wait_job_running(queue2_job_id)
     # check queue1 AMIs are not replaced
     _check_queue_ami(cluster, ec2, pcluster_ami_id, "queue1")