From 8f3cdc15e60e5b55d7faa265ae0e6110fbe612d3 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 18 Aug 2026 16:32:04 -0400 Subject: [PATCH 1/6] [Test] test_patching_cluster: add crc32_generic to lazily-loaded kernel modules on every head node to prevent false alarms. --- .../tests/patching/test_patching.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index 17df4dff93..63c9670d61 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -55,37 +55,37 @@ # Maintained per OS and per node type. Keep module names alphabetically sorted. LAZY_KERNEL_MODULES = { "alinux2023": { - HEAD_NODE: ["tls"], + HEAD_NODE: ["crc32_generic", "tls"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rhel8": { - HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rhel9": { - HEAD_NODE: ["tls"], + HEAD_NODE: ["crc32_generic", "tls"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rocky8": { - HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rocky9": { - HEAD_NODE: ["tls"], + HEAD_NODE: ["crc32_generic", "tls"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "ubuntu2204": { - HEAD_NODE: ["tls"], + HEAD_NODE: ["crc32_generic", "tls"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "ubuntu2404": { - HEAD_NODE: ["tls"], + HEAD_NODE: ["crc32_generic", "tls"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, From dd6cd20d129cf043a4fe64a794666741bac238fb Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 18 Aug 2026 16:32:24 -0400 Subject: [PATCH 2/6] [Test] test_patching_cluster: deduplicate head-node lazy kernel modules via COMMON_HEAD_NODE_LAZY_MODULES to avoid code duplication. --- .../tests/patching/test_patching.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index 63c9670d61..ace2f3beee 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -53,9 +53,11 @@ # may or may not have run by the time we snapshot). We force-load these after patching so that # the before/after comparison does not flag them as missing. # Maintained per OS and per node type. Keep module names alphabetically sorted. +# Modules tolerated on the head node across all OSes. +COMMON_HEAD_NODE_LAZY_MODULES = ["crc32_generic", "tls"] LAZY_KERNEL_MODULES = { "alinux2023": { - HEAD_NODE: ["crc32_generic", "tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, @@ -65,7 +67,7 @@ LOGIN_NODE: ["tls"], }, "rhel9": { - HEAD_NODE: ["crc32_generic", "tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, @@ -75,17 +77,17 @@ LOGIN_NODE: ["tls"], }, "rocky9": { - HEAD_NODE: ["crc32_generic", "tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "ubuntu2204": { - HEAD_NODE: ["crc32_generic", "tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "ubuntu2404": { - HEAD_NODE: ["crc32_generic", "tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, From 15024e72290457aebcfdb6ca0036240f4739b556 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 18 Aug 2026 16:33:00 -0400 Subject: [PATCH 3/6] [Test] test_build_image_no_internet: prevent Proxy verification timeout by decoupling ProxyVerificationWaitCondition from the Proxy instance. Also reduced the risk of proxy infra deployment failure by increasing the timeout of the proxy instance to not fail on slow user data. --- cloudformation/proxy/proxy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index 61fa620952..e4e8f58f8d 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -623,11 +623,11 @@ Resources: ProxyVerificationWaitCondition: Type: AWS::CloudFormation::WaitCondition DependsOn: - - Proxy + - ProxyClient Properties: Count: 1 Handle: !Ref ProxyVerificationWaitConditionHandle - Timeout: 600 + Timeout: 900 Outputs: From 38791fa715994b2a7f42abbe29f853b91d15073a Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 18 Aug 2026 17:42:32 -0400 Subject: [PATCH 4/6] [Test] test_cluster_with_gpu_health_checks: prevent ICE by reserving all the instances required by the test and let the cluster use the specific subnet where they have been reserved. --- tests/integration-tests/configs/develop.yaml | 2 +- .../test_cluster_with_gpu_health_checks/pcluster.config.yaml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/integration-tests/configs/develop.yaml b/tests/integration-tests/configs/develop.yaml index d179741fc4..0d80cac27e 100644 --- a/tests/integration-tests/configs/develop.yaml +++ b/tests/integration-tests/configs/develop.yaml @@ -322,7 +322,7 @@ test-suites: health_checks: test_gpu_health_checks.py::test_cluster_with_gpu_health_checks: dimensions: - - regions: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0_CAPACITY_RESERVATION_3_INSTANCES_1_HOURS_NOPG_OS_X86_5 }}] + - regions: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0_CAPACITY_RESERVATION_4_INSTANCES_1_HOURS_NOPG_OS_X86_5__c5_xlarge_CAPACITY_RESERVATION_5_INSTANCES_1_HOURS_NOPG_OS_X86_5 }}] instances: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0 }}] oss: [{{ OS_X86_5 }}] schedulers: ["slurm"] diff --git a/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml b/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml index 828edf26af..09f36dacf6 100644 --- a/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml +++ b/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml @@ -93,9 +93,7 @@ Scheduling: - InstanceType: {{ non_gpu_instance }} Networking: SubnetIds: - {% for private_subnet_id in private_subnet_ids %} - {{ private_subnet_id }} - {% endfor %} - Name: queue-2 ComputeResources: - Name: compute-resource-1 @@ -112,7 +110,5 @@ Scheduling: Enabled: true Networking: SubnetIds: - {% for private_subnet_id in private_subnet_ids %} - {{ private_subnet_id }} - {% endfor %} {% endif %} From 63cdfe45db66c6a5e7f6bd55b5c29b6941285108 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 14 Aug 2026 17:54:51 -0400 Subject: [PATCH 5/6] [Test] Retry EFS mount assertion for IAM-authorized EFS in test_multiple_efs. IAM policy evaluation can transiently delay the mount on head node reboot, causing flaky failures. Retry the mount check only for IAM-authorized EFS. --- .../tests/storage/test_efs.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/integration-tests/tests/storage/test_efs.py b/tests/integration-tests/tests/storage/test_efs.py index 54ebda4ba3..c635efe8b3 100644 --- a/tests/integration-tests/tests/storage/test_efs.py +++ b/tests/integration-tests/tests/storage/test_efs.py @@ -17,6 +17,8 @@ from assertpy import assert_that from cfn_stacks_factory import CfnVpcStack from remote_command_executor import RemoteCommandExecutor +from retrying import retry +from time_utils import minutes, seconds from utils import get_arn_partition, get_compute_nodes_instance_ips from tests.common.utils import get_sts_endpoint, reboot_head_node @@ -350,12 +352,22 @@ def _check_efs_correctly_mounted_and_shared( all_mount_dirs, remote_command_executor, scheduler_commands, iam_authorizations, encryption_in_transits ): for i, mount_dir in enumerate(all_mount_dirs): - test_efs_correctly_mounted( - remote_command_executor, - mount_dir, - encryption_in_transits[i], - iam_authorizations[i], - ) + if iam_authorizations[i]: + # An EFS with IAM authorization can be transiently slower to mount because the mount depends on + # IAM policy evaluation. As such, we retry the assertion to prevent flaky test failures. + retry(wait_fixed=seconds(10), stop_max_delay=minutes(3))(test_efs_correctly_mounted)( + remote_command_executor, + mount_dir, + encryption_in_transits[i], + iam_authorizations[i], + ) + else: + test_efs_correctly_mounted( + remote_command_executor, + mount_dir, + encryption_in_transits[i], + iam_authorizations[i], + ) _test_efs_correctly_shared(remote_command_executor, mount_dir, scheduler_commands) From 98e70dcd311e09019ab8b8bfe6f83e624836ee26 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 14 Aug 2026 17:55:51 -0400 Subject: [PATCH 6/6] [Test] Fix flaky test_queue_parameters_update: assert TERMINATE requeue deterministically The TERMINATE branch snapshotted the transient CONFIGURING state via a single-shot assert_job_state after a fixed sleep(10). When replacement nodes boot quickly the whole node swap finishes inside the update-cluster --wait window, so the job is already RUNNING again by the time the test samples it, causing false failures. Assert the deterministic outcome instead: the job was requeued (wait_job_requeued) and is running again, and the original instance it ran on was terminated (wait_instance_replaced_or_terminating). Add SlurmCommands.get_job_instance_id to resolve the job's instance via Slurm, and wait_job_requeued helper. --- .../tests/common/schedulers_common.py | 15 +++++++++++++ .../tests/update/test_update.py | 21 ++++++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/integration-tests/tests/common/schedulers_common.py b/tests/integration-tests/tests/common/schedulers_common.py index 9b8a37af8d..3b454764de 100644 --- a/tests/integration-tests/tests/common/schedulers_common.py +++ b/tests/integration-tests/tests/common/schedulers_common.py @@ -496,6 +496,15 @@ def get_node_addr(self, node_name): node_addr = re.search(r"NodeAddr=(.*) NodeHostName", result).group(1) return node_addr + def get_job_instance_id(self, job_id): + """Return the id of the EC2 instance the job is running on, as reported by Slurm.""" + node_name = self._remote_command_executor.run_remote_command( + f'scontrol show jobs {job_id} --json | jq -r ".jobs[0].batch_host"' + ).stdout.strip() + return self._remote_command_executor.run_remote_command( + f'scontrol show nodes {node_name} --json | jq -r ".nodes[0].instance_id"' + ).stdout.strip() + def submit_command_and_assert_job_accepted(self, submit_command_args): """Submit a command and assert the job is accepted by scheduler.""" result = self.submit_command(**submit_command_args) @@ -520,6 +529,12 @@ def wait_job_running(self, job_id): result = self._remote_command_executor.run_remote_command("scontrol show jobs -o {0}".format(job_id)) assert_that(result.stdout).contains("JobState=RUNNING") + @retry(wait_fixed=seconds(10), stop_max_delay=minutes(13)) + def wait_job_requeued(self, job_id, times=1): + """Wait till the job has been requeued at least `times` times (Restarts>=times).""" + restarts = self.get_job_info(job_id, field="Restarts") + assert_that(int(restarts)).is_greater_than_or_equal_to(times) + def get_node_info(self, nodename): """Get node info.""" return self._remote_command_executor.run_remote_command("scontrol show nodes {0}".format(nodename)).stdout diff --git a/tests/integration-tests/tests/update/test_update.py b/tests/integration-tests/tests/update/test_update.py index 9aca180d36..0b5fc47b5d 100644 --- a/tests/integration-tests/tests/update/test_update.py +++ b/tests/integration-tests/tests/update/test_update.py @@ -42,7 +42,12 @@ wait_for_computefleet_changed, ) -from tests.common.assertions import assert_instance_config_version_on_ddb, assert_lines_in_logs, assert_no_msg_in_logs +from tests.common.assertions import ( + assert_instance_config_version_on_ddb, + assert_lines_in_logs, + assert_no_msg_in_logs, + wait_instance_replaced_or_terminating, +) from tests.common.hit_common import ( assert_compute_node_states, assert_initial_conditions, @@ -888,6 +893,10 @@ def _test_update_queue_strategy_with_running_job( scheduler_commands.wait_job_running(queue2_job_id) logging.info(f"Job {queue2_job_id} is running on queue2") + # Ask Slurm exactly which instance the queue2 job is running on, so that after a TERMINATE update + # we can assert that exact instance was terminated (the job's node was replaced). + queue2_instance_id = scheduler_commands.get_job_instance_id(queue2_job_id) + logging.info(f"Updating cluster with strategy {queue_update_strategy} with running jobs") updated_config_file = pcluster_config_reader( config_file="pcluster.config.update_with_running_job.yaml", @@ -913,8 +922,14 @@ def _test_update_queue_strategy_with_running_job( queue1_nodes = scheduler_commands.get_compute_nodes("queue1") assert_compute_node_states(scheduler_commands, queue1_nodes, expected_states=["mixed", "allocated"]) if queue_update_strategy == "TERMINATE": - time.sleep(10) - scheduler_commands.assert_job_state(queue2_job_id, "CONFIGURING") + # TERMINATE forcibly terminates queue2's compute nodes and requeues the running job onto + # freshly launched replacement nodes. Assert the deterministic outcome (the job was requeued + # and is running again on the replaced instances) rather than snapshotting the transient + # CONFIGURING state, which can close before we sample it when replacement nodes boot quickly + # (the whole swap can finish inside the update-cluster --wait window). + scheduler_commands.wait_job_requeued(queue2_job_id) + wait_instance_replaced_or_terminating(queue2_instance_id, region) + scheduler_commands.wait_job_running(queue2_job_id) # check queue1 AMIs are not replaced _check_queue_ami(cluster, ec2, pcluster_ami_id, "queue1")