diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index 61fa620952..e4e8f58f8d 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -623,11 +623,11 @@ Resources: ProxyVerificationWaitCondition: Type: AWS::CloudFormation::WaitCondition DependsOn: - - Proxy + - ProxyClient Properties: Count: 1 Handle: !Ref ProxyVerificationWaitConditionHandle - Timeout: 600 + Timeout: 900 Outputs: diff --git a/tests/integration-tests/configs/develop.yaml b/tests/integration-tests/configs/develop.yaml index d179741fc4..0d80cac27e 100644 --- a/tests/integration-tests/configs/develop.yaml +++ b/tests/integration-tests/configs/develop.yaml @@ -322,7 +322,7 @@ test-suites: health_checks: test_gpu_health_checks.py::test_cluster_with_gpu_health_checks: dimensions: - - regions: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0_CAPACITY_RESERVATION_3_INSTANCES_1_HOURS_NOPG_OS_X86_5 }}] + - regions: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0_CAPACITY_RESERVATION_4_INSTANCES_1_HOURS_NOPG_OS_X86_5__c5_xlarge_CAPACITY_RESERVATION_5_INSTANCES_1_HOURS_NOPG_OS_X86_5 }}] instances: [{{ US_WEST_2_GPU_INSTANCE_TYPE_0 }}] oss: [{{ OS_X86_5 }}] schedulers: ["slurm"] diff --git a/tests/integration-tests/tests/common/schedulers_common.py b/tests/integration-tests/tests/common/schedulers_common.py index 9b8a37af8d..3b454764de 100644 --- a/tests/integration-tests/tests/common/schedulers_common.py +++ b/tests/integration-tests/tests/common/schedulers_common.py @@ -496,6 +496,15 @@ def get_node_addr(self, node_name): node_addr = re.search(r"NodeAddr=(.*) NodeHostName", result).group(1) return node_addr + def get_job_instance_id(self, job_id): + """Return the id of the EC2 instance the job is running on, as reported by Slurm.""" + node_name = self._remote_command_executor.run_remote_command( + f'scontrol show jobs {job_id} --json | jq -r ".jobs[0].batch_host"' + ).stdout.strip() + return self._remote_command_executor.run_remote_command( + f'scontrol show nodes {node_name} --json | jq -r ".nodes[0].instance_id"' + ).stdout.strip() + def submit_command_and_assert_job_accepted(self, submit_command_args): """Submit a command and assert the job is accepted by scheduler.""" result = self.submit_command(**submit_command_args) @@ -520,6 +529,12 @@ def wait_job_running(self, job_id): result = self._remote_command_executor.run_remote_command("scontrol show jobs -o {0}".format(job_id)) assert_that(result.stdout).contains("JobState=RUNNING") + @retry(wait_fixed=seconds(10), stop_max_delay=minutes(13)) + def wait_job_requeued(self, job_id, times=1): + """Wait till the job has been requeued at least `times` times (Restarts>=times).""" + restarts = self.get_job_info(job_id, field="Restarts") + assert_that(int(restarts)).is_greater_than_or_equal_to(times) + def get_node_info(self, nodename): """Get node info.""" return self._remote_command_executor.run_remote_command("scontrol show nodes {0}".format(nodename)).stdout diff --git a/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml b/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml index 828edf26af..09f36dacf6 100644 --- a/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml +++ b/tests/integration-tests/tests/health_checks/test_gpu_health_checks/test_cluster_with_gpu_health_checks/pcluster.config.yaml @@ -93,9 +93,7 @@ Scheduling: - InstanceType: {{ non_gpu_instance }} Networking: SubnetIds: - {% for private_subnet_id in private_subnet_ids %} - {{ private_subnet_id }} - {% endfor %} - Name: queue-2 ComputeResources: - Name: compute-resource-1 @@ -112,7 +110,5 @@ Scheduling: Enabled: true Networking: SubnetIds: - {% for private_subnet_id in private_subnet_ids %} - {{ private_subnet_id }} - {% endfor %} {% endif %} diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index 17df4dff93..ace2f3beee 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -53,39 +53,41 @@ # may or may not have run by the time we snapshot). We force-load these after patching so that # the before/after comparison does not flag them as missing. # Maintained per OS and per node type. Keep module names alphabetically sorted. +# Modules tolerated on the head node across all OSes. +COMMON_HEAD_NODE_LAZY_MODULES = ["crc32_generic", "tls"] LAZY_KERNEL_MODULES = { "alinux2023": { - HEAD_NODE: ["tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rhel8": { - HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rhel9": { - HEAD_NODE: ["tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rocky8": { - HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"], COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "rocky9": { - HEAD_NODE: ["tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "ubuntu2204": { - HEAD_NODE: ["tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, "ubuntu2404": { - HEAD_NODE: ["tls"], + HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, COMPUTE_NODE: ["tls"], LOGIN_NODE: ["tls"], }, diff --git a/tests/integration-tests/tests/storage/test_efs.py b/tests/integration-tests/tests/storage/test_efs.py index 54ebda4ba3..c635efe8b3 100644 --- a/tests/integration-tests/tests/storage/test_efs.py +++ b/tests/integration-tests/tests/storage/test_efs.py @@ -17,6 +17,8 @@ from assertpy import assert_that from cfn_stacks_factory import CfnVpcStack from remote_command_executor import RemoteCommandExecutor +from retrying import retry +from time_utils import minutes, seconds from utils import get_arn_partition, get_compute_nodes_instance_ips from tests.common.utils import get_sts_endpoint, reboot_head_node @@ -350,12 +352,22 @@ def _check_efs_correctly_mounted_and_shared( all_mount_dirs, remote_command_executor, scheduler_commands, iam_authorizations, encryption_in_transits ): for i, mount_dir in enumerate(all_mount_dirs): - test_efs_correctly_mounted( - remote_command_executor, - mount_dir, - encryption_in_transits[i], - iam_authorizations[i], - ) + if iam_authorizations[i]: + # An EFS with IAM authorization can be transiently slower to mount because the mount depends on + # IAM policy evaluation. As such, we retry the assertion to prevent flaky test failures. + retry(wait_fixed=seconds(10), stop_max_delay=minutes(3))(test_efs_correctly_mounted)( + remote_command_executor, + mount_dir, + encryption_in_transits[i], + iam_authorizations[i], + ) + else: + test_efs_correctly_mounted( + remote_command_executor, + mount_dir, + encryption_in_transits[i], + iam_authorizations[i], + ) _test_efs_correctly_shared(remote_command_executor, mount_dir, scheduler_commands) diff --git a/tests/integration-tests/tests/update/test_update.py b/tests/integration-tests/tests/update/test_update.py index 9aca180d36..0b5fc47b5d 100644 --- a/tests/integration-tests/tests/update/test_update.py +++ b/tests/integration-tests/tests/update/test_update.py @@ -42,7 +42,12 @@ wait_for_computefleet_changed, ) -from tests.common.assertions import assert_instance_config_version_on_ddb, assert_lines_in_logs, assert_no_msg_in_logs +from tests.common.assertions import ( + assert_instance_config_version_on_ddb, + assert_lines_in_logs, + assert_no_msg_in_logs, + wait_instance_replaced_or_terminating, +) from tests.common.hit_common import ( assert_compute_node_states, assert_initial_conditions, @@ -888,6 +893,10 @@ def _test_update_queue_strategy_with_running_job( scheduler_commands.wait_job_running(queue2_job_id) logging.info(f"Job {queue2_job_id} is running on queue2") + # Ask Slurm exactly which instance the queue2 job is running on, so that after a TERMINATE update + # we can assert that exact instance was terminated (the job's node was replaced). + queue2_instance_id = scheduler_commands.get_job_instance_id(queue2_job_id) + logging.info(f"Updating cluster with strategy {queue_update_strategy} with running jobs") updated_config_file = pcluster_config_reader( config_file="pcluster.config.update_with_running_job.yaml", @@ -913,8 +922,14 @@ def _test_update_queue_strategy_with_running_job( queue1_nodes = scheduler_commands.get_compute_nodes("queue1") assert_compute_node_states(scheduler_commands, queue1_nodes, expected_states=["mixed", "allocated"]) if queue_update_strategy == "TERMINATE": - time.sleep(10) - scheduler_commands.assert_job_state(queue2_job_id, "CONFIGURING") + # TERMINATE forcibly terminates queue2's compute nodes and requeues the running job onto + # freshly launched replacement nodes. Assert the deterministic outcome (the job was requeued + # and is running again on the replaced instances) rather than snapshotting the transient + # CONFIGURING state, which can close before we sample it when replacement nodes boot quickly + # (the whole swap can finish inside the update-cluster --wait window). + scheduler_commands.wait_job_requeued(queue2_job_id) + wait_instance_replaced_or_terminating(queue2_instance_id, region) + scheduler_commands.wait_job_running(queue2_job_id) # check queue1 AMIs are not replaced _check_queue_ami(cluster, ec2, pcluster_ami_id, "queue1")