diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index aee64a39..3b656f65 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -526,6 +526,14 @@ var testTypeToTestConfig = map[string][]testConfig{ ami: "AL2023_x86_64_STANDARD", k8sVersion: "1.35", }, + { + testDir: "./test/otel/multi_efa_dra", + terraformDir: "terraform/eks/daemon/otel-multi-efa-dra", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "c6in.32xlarge", + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, { testDir: "./test/otel/neuron", terraformDir: "terraform/eks/daemon/otel-neuron", @@ -534,6 +542,22 @@ var testTypeToTestConfig = map[string][]testConfig{ ami: "AL2023_x86_64_NEURON", k8sVersion: "1.35", }, + { + testDir: "./test/otel/neuron_dra", + terraformDir: "terraform/eks/daemon/otel-neuron-dra", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "trn1.2xlarge", + ami: "AL2023_x86_64_NEURON", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/gpu_dra", + terraformDir: "terraform/eks/daemon/otel-gpu-dra", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "g4dn.12xlarge", + ami: "AL2023_x86_64_NVIDIA", + k8sVersion: "1.35", + }, { testDir: "./test/otel/performance", terraformDir: "terraform/eks/daemon/otel-performance", diff --git a/terraform/eks/daemon/otel-gpu-dra/main.tf b/terraform/eks/daemon/otel-gpu-dra/main.tf new file mode 100644 index 00000000..c1bd650b --- /dev/null +++ b/terraform/eks/daemon/otel-gpu-dra/main.tf @@ -0,0 +1,350 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +# GPU DRA-path integration test cluster. Mirrors otel-gpu but exposes GPUs via the +# NVIDIA DRA driver (DeviceClass gpu.nvidia.com) instead of the device plugin, and +# the burn workload claims a GPU via a ResourceClaimTemplate. One multi-GPU node +# (g4dn.12xlarge = 4 T4) proves per-device DRA correlation (1 claimed GPU -> the +# burn pod, the other 3 uncorrelated). + +module "common" { + source = "../../../common" + cwagent_image_repo = var.cwagent_image_repo + cwagent_image_tag = var.cwagent_image_tag +} + +module "basic_components" { + source = "../../../basic_components" + region = var.region +} + +locals { + aws_eks = "aws eks --region ${var.region}" +} + +resource "aws_eks_cluster" "this" { + name = "cwagent-eks-integ-${module.common.testing_id}" + role_arn = module.basic_components.role_arn + version = var.k8s_version + vpc_config { + subnet_ids = module.basic_components.public_subnet_ids + security_group_ids = [module.basic_components.security_group] + } +} + +# --- IAM --- + +resource "aws_iam_role" "node_role" { + name = "cwagent-otel-gpu-dra-Worker-Role-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEKSWorkerNodePolicy" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEKS_CNI_Policy" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEC2ContainerRegistryReadOnly" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role" "pod_identity_role" { + name = "cwagent-otel-gpu-dra-pod-identity-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) +} + +resource "aws_iam_role_policy_attachment" "pod_identity_CloudWatchAgentServerPolicy" { + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" + role = aws_iam_role.pod_identity_role.name +} + +# --- Node Groups --- + +resource "aws_eks_node_group" "standard" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "standard-${module.common.testing_id}" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = module.basic_components.public_subnet_ids + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + ami_type = "AL2023_x86_64_STANDARD" + capacity_type = "ON_DEMAND" + disk_size = 20 + instance_types = ["t3.medium"] + + depends_on = [ + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, + ] +} + +# Multi-GPU node (g4dn.12xlarge, 4 GPUs). No taint: dedicated by being the only GPU +# node; the DRA driver DaemonSet + burn pod land here via scheduling/nodeSelector. +resource "aws_eks_node_group" "gpu_multi" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "gpu-multi-${module.common.testing_id}" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = module.basic_components.public_subnet_ids + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + ami_type = var.ami_type + capacity_type = "ON_DEMAND" + disk_size = 40 + instance_types = [var.instance_type] + + labels = { + "nvidia.com/gpu.present" = "true" + "ci-test.example.com/node-color" = "green" + } + + depends_on = [ + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, + ] +} + +# --- EKS Addon: Pod Identity agent --- + +resource "aws_eks_addon" "pod_identity_agent" { + depends_on = [aws_eks_node_group.standard] + cluster_name = aws_eks_cluster.this.name + addon_name = "eks-pod-identity-agent" +} + +# --- Update kubeconfig --- + +resource "null_resource" "kubectl" { + depends_on = [aws_eks_cluster.this, aws_eks_node_group.standard, aws_eks_node_group.gpu_multi] + provisioner "local-exec" { + command = "${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name}" + } +} + +# --- NVIDIA DRA driver (Helm) — replaces the device plugin --- +# +# NVIDIA DRA driver install, tuned for a single-node whole-GPU EKS cluster. Verified +# against chart 25.12.0: +# - resources.gpus.enabled=true turns on whole-GPU allocation (DeviceClass +# gpu.nvidia.com), but the chart hard-guards it behind gpuResourcesEnabledOverride +# =true (it refuses to co-exist with the standard GPU device plugin until KEP 5004 +# is GA). We do not run the device plugin, so the override is safe and required — +# without it helm template fails a validation.yaml assertion. +# - resources.computeDomains.enabled=false drops the ComputeDomain controller, whose +# nodeAffinity requires node-role.kubernetes.io/control-plane. EKS has no such +# (customer-visible) nodes, so the controller would stay Pending and, with helm's +# default wait=true, time out the apply. We only need single-node whole-GPU +# correlation, so ComputeDomains (multi-node GPU/IMEX) is unnecessary. +# - kubeletPlugin.nodeSelector pins the plugin DaemonSet to the GPU node. The chart's +# default kubeletPlugin nodeAffinity already ORs in nvidia.com/gpu.present=true +# (which our nodegroup sets), so no affinity clearing is needed on this chart. +# After apply, `kubectl get deviceclass gpu.nvidia.com` and a ResourceSlice for the +# node should exist. +resource "helm_release" "nvidia_dra_driver" { + depends_on = [aws_eks_node_group.gpu_multi, null_resource.kubectl] + + name = "nvidia-dra-driver-gpu" + repository = var.nvidia_dra_repo + chart = "nvidia-dra-driver-gpu" + namespace = "nvidia-dra-driver-gpu" + create_namespace = true + version = var.nvidia_dra_chart_version != "" ? var.nvidia_dra_chart_version : null + + set = [ + { name = "gpuResourcesEnabledOverride", value = "true" }, + { name = "resources.gpus.enabled", value = "true" }, + { name = "resources.computeDomains.enabled", value = "false" }, + # nodeSelector map values must be strings; type=string stops the provider + # coercing "true" to a bool (which fails DaemonSet unmarshalling). + { name = "kubeletPlugin.nodeSelector.nvidia\\.com/gpu\\.present", value = "true", type = "string" }, + ] +} + +# --- multi-gpu-burn-dra Deployment: claims 1 GPU via DRA --- + +resource "null_resource" "gpu_burn_dra" { + depends_on = [helm_release.nvidia_dra_driver, null_resource.kubectl] + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF' | kubectl apply -f - + apiVersion: resource.k8s.io/v1 + kind: ResourceClaimTemplate + metadata: + name: gpu-1 + namespace: default + spec: + spec: + devices: + requests: + - name: gpu + exactly: + deviceClassName: gpu.nvidia.com + count: 1 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: gpu-burn-dra + namespace: default + spec: + replicas: 1 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 300 + selector: + matchLabels: + app: gpu-burn-dra + template: + metadata: + labels: + app: gpu-burn-dra + ci-test.example.com/pod-color: magenta + spec: + nodeSelector: + node.kubernetes.io/instance-type: ${var.instance_type} + resourceClaims: + - name: gpu + resourceClaimTemplateName: gpu-1 + containers: + - name: gpu-burn + image: chrstnhntschl/gpu_burn:latest + args: ["3600"] + resources: + claims: + - name: gpu + EOF + EOT + } +} + +# --- Helm chart install (observability) --- + +data "external" "clone_helm_chart" { + program = ["bash", "-c", <<-EOT + rm -rf ./helm-charts + git clone -b ${var.helm_chart_branch} ${var.helm_chart_repo_url} ./helm-charts + echo '{"status":"ready"}' + EOT + ] +} + +resource "helm_release" "aws_observability" { + name = "amazon-cloudwatch-observability" + chart = "./helm-charts/charts/amazon-cloudwatch-observability" + namespace = "amazon-cloudwatch" + create_namespace = true + wait = false + timeout = 600 + + set = [ + { name = "clusterName", value = aws_eks_cluster.this.name }, + { name = "region", value = var.region }, + { name = "otelContainerInsights.enabled", value = "true" }, + ] + + depends_on = [ + aws_eks_addon.pod_identity_agent, + null_resource.kubectl, + data.external.clone_helm_chart, + ] +} + +# --- Pod Identity association --- + +resource "aws_eks_pod_identity_association" "cloudwatch_agent" { + depends_on = [helm_release.aws_observability] + cluster_name = aws_eks_cluster.this.name + namespace = "amazon-cloudwatch" + service_account = "cloudwatch-agent" + role_arn = aws_iam_role.pod_identity_role.arn +} + +# --- Patch agent image --- + +resource "null_resource" "update_image" { + depends_on = [helm_release.aws_observability, null_resource.kubectl] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + sleep 30 + kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' \ + -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' + kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent-cluster-scraper --type='json' \ + -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' 2>/dev/null || true + sleep 10 + EOT + } +} + +# --- Restart pods --- + +resource "null_resource" "restart_pods" { + depends_on = [aws_eks_pod_identity_association.cloudwatch_agent, null_resource.update_image] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + kubectl -n amazon-cloudwatch rollout restart daemonset/cloudwatch-agent + kubectl -n amazon-cloudwatch rollout restart deployment/cloudwatch-agent-cluster-scraper 2>/dev/null || true + kubectl -n amazon-cloudwatch rollout status daemonset/cloudwatch-agent --timeout=120s + EOT + } +} + +# --- Test runner --- + +resource "null_resource" "validator" { + depends_on = [null_resource.restart_pods, null_resource.gpu_burn_dra] + triggers = { always_run = timestamp() } + provisioner "local-exec" { + command = <<-EOT + echo "Running OTEL GPU DRA cluster integration tests" + cd ../../../.. + + echo "Waiting for dcgm-exporter pods to be ready..." + for i in $(seq 1 30); do + READY=$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=dcgm-exporter -o jsonpath='{.items[*].status.phase}' 2>/dev/null | tr ' ' '\n' | grep -c Running || true) + if [ "$READY" -ge 1 ] 2>/dev/null; then break; fi + sleep 20 + done + + echo "Waiting 6 minutes for GPU metrics to propagate (Zeus 5-min staleness window)..." + sleep 360 + + go test -tags integration -timeout 1h -v ${var.test_dir} \ + -eksClusterName=${aws_eks_cluster.this.name} \ + -computeType=EKS \ + -eksDeploymentStrategy=DAEMON \ + -region=${var.region} + EOT + } +} diff --git a/terraform/eks/daemon/otel-gpu-dra/providers.tf b/terraform/eks/daemon/otel-gpu-dra/providers.tf new file mode 100644 index 00000000..a562a7f1 --- /dev/null +++ b/terraform/eks/daemon/otel-gpu-dra/providers.tf @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +provider "aws" { + region = var.region +} + +provider "kubernetes" { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } +} + +provider "helm" { + kubernetes = { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } + } +} diff --git a/terraform/eks/daemon/otel-gpu-dra/variables.tf b/terraform/eks/daemon/otel-gpu-dra/variables.tf new file mode 100644 index 00000000..1c689d17 --- /dev/null +++ b/terraform/eks/daemon/otel-gpu-dra/variables.tf @@ -0,0 +1,66 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +variable "region" { + type = string + default = "us-west-2" +} + +variable "test_dir" { + type = string + default = "./test/otel/gpu_dra" +} + +variable "cwagent_image_repo" { + type = string + default = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" +} + +variable "cwagent_image_tag" { + type = string + default = "latest" +} + +# The chart must render the DRA correlation config (dra_device_types incl. the +# gpu-dra entry keyed on the gpu.nvidia.com driver) and grant the agent +# ServiceAccount resource.k8s.io RBAC. Override to a fork/branch until that lands +# on main (helm-charts PR #356). +variable "helm_chart_branch" { + type = string + default = "main" +} + +variable "helm_chart_repo_url" { + type = string + default = "https://github.com/aws-observability/helm-charts.git" +} + +# The awsdevicepodcorrelation processor uses the GA resource.k8s.io/v1 DRA API +# (available since Kubernetes 1.34), so the cluster must be 1.34+. +variable "k8s_version" { + type = string + default = "1.35" +} + +variable "ami_type" { + type = string + default = "AL2023_x86_64_NVIDIA" +} + +# Multi-GPU node: g4dn.12xlarge = 4 T4 GPUs. +variable "instance_type" { + type = string + default = "g4dn.12xlarge" +} + +# NVIDIA DRA driver Helm chart (DeviceClass gpu.nvidia.com). +# Repo: https://helm.ngc.nvidia.com/nvidia, chart: nvidia-dra-driver-gpu. +variable "nvidia_dra_repo" { + type = string + default = "https://helm.ngc.nvidia.com/nvidia" +} + +variable "nvidia_dra_chart_version" { + type = string + default = "" +} diff --git a/terraform/eks/daemon/otel-multi-efa-dra/.gitignore b/terraform/eks/daemon/otel-multi-efa-dra/.gitignore new file mode 100644 index 00000000..7a0bdf43 --- /dev/null +++ b/terraform/eks/daemon/otel-multi-efa-dra/.gitignore @@ -0,0 +1,3 @@ +*.tfstate* +.terraform/ +helm-charts/ diff --git a/terraform/eks/daemon/otel-multi-efa-dra/cloudwatch-pod-identity.tf b/terraform/eks/daemon/otel-multi-efa-dra/cloudwatch-pod-identity.tf new file mode 100644 index 00000000..a09bf5a0 --- /dev/null +++ b/terraform/eks/daemon/otel-multi-efa-dra/cloudwatch-pod-identity.tf @@ -0,0 +1,60 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +# IAM role for CloudWatch Observability addon +resource "aws_iam_role" "cloudwatch_observability" { + name = "cloudwatch-observability-${module.common.testing_id}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "pods.eks.amazonaws.com" + } + Action = [ + "sts:AssumeRole", + "sts:TagSession" + ] + } + ] + }) + + tags = { + Name = "cloudwatch-observability-${module.common.testing_id}" + Owner = "default" + } +} + +# Attach CloudWatch policies +resource "aws_iam_role_policy_attachment" "cloudwatch_observability_server" { + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" + role = aws_iam_role.cloudwatch_observability.name +} + +# Pod Identity association for CloudWatch agent +resource "aws_eks_pod_identity_association" "cloudwatch_agent" { + cluster_name = module.eks.cluster_name + namespace = "amazon-cloudwatch" + service_account = "cloudwatch-agent" + role_arn = aws_iam_role.cloudwatch_observability.arn + + tags = { + Name = "cloudwatch-agent-${module.common.testing_id}" + Owner = "default" + } +} + +# Pod Identity association for Fluent Bit +resource "aws_eks_pod_identity_association" "fluent_bit" { + cluster_name = module.eks.cluster_name + namespace = "amazon-cloudwatch" + service_account = "fluent-bit" + role_arn = aws_iam_role.cloudwatch_observability.arn + + tags = { + Name = "fluent-bit-${module.common.testing_id}" + Owner = "default" + } +} diff --git a/terraform/eks/daemon/otel-multi-efa-dra/main.tf b/terraform/eks/daemon/otel-multi-efa-dra/main.tf new file mode 100644 index 00000000..798f5ea5 --- /dev/null +++ b/terraform/eks/daemon/otel-multi-efa-dra/main.tf @@ -0,0 +1,244 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +locals { + aws_eks = "aws eks --region ${var.region}" +} + +module "common" { + source = "../../../common" + cwagent_image_repo = var.cwagent_image_repo + cwagent_image_tag = var.cwagent_image_tag +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" + + name = "cwagent-eks-integ-${module.common.testing_id}" + kubernetes_version = var.k8s_version + + # Full role name (<=64); the module's "-cluster-" name_prefix would exceed the 38-char cap. + iam_role_use_name_prefix = false + + vpc_id = aws_vpc.efa_test_vpc.id + subnet_ids = aws_subnet.efa_test_public_subnet[*].id + + endpoint_public_access = true + enable_cluster_creator_admin_permissions = true + + eks_managed_node_groups = { + standard = { + ami_type = "AL2023_x86_64_STANDARD" + instance_types = ["t3.medium"] + min_size = 1 + max_size = 1 + desired_size = 1 + subnet_ids = aws_subnet.efa_test_public_subnet[*].id + } + + multi_efa = { + # enable_efa_support wires EFA network interfaces into the launch template. + # It does NOT install the EFA k8s device plugin (that is a separate helm + # release on the device-plugin path); here EFA is exposed via DRA instead. + enable_efa_support = true + ami_type = var.ami_type + instance_types = [var.instance_type] + min_size = 1 + max_size = 1 + desired_size = 1 + subnet_ids = aws_subnet.efa_test_private_subnet[*].id + + labels = { + "ci-test.example.com/multi-efa-dra" = "true" + "ci-test.example.com/node-color" = "yellow" + } + } + } + + addons = { + coredns = {} + eks-pod-identity-agent = { before_compute = true } + kube-proxy = {} + vpc-cni = { before_compute = true } + } + + tags = { Owner = "default" } +} + +# --- DRA driver (dranet) --- +# Replaces the EFA device plugin: exposes the node's EFA interfaces as DRA +# devices under the DeviceClass efa.networking.k8s.aws (driver name "dra.net") +# and publishes a ResourceSlice carrying the dra.net/rdmaDevice attribute that +# the processor keys on. dranet and the EFA device plugin must not both manage +# EFA on the same node, so the device plugin is intentionally not installed here. + +resource "helm_release" "aws_dranet" { + name = "aws-dranet" + repository = "https://aws.github.io/eks-charts" + chart = "aws-dranet" + version = var.dranet_version + namespace = "kube-system" + wait = true + + depends_on = [module.eks] +} + +# --- Helm chart install --- + +resource "null_resource" "kubectl" { + depends_on = [module.eks] + provisioner "local-exec" { + command = "${local.aws_eks} update-kubeconfig --name ${module.eks.cluster_name}" + } +} + +data "external" "clone_helm_chart" { + program = ["bash", "-c", <<-EOT + rm -rf ./helm-charts + git clone -b ${var.helm_chart_branch} ${var.helm_chart_repo_url} ./helm-charts + echo '{"status":"ready"}' + EOT + ] +} + +resource "helm_release" "aws_observability" { + name = "amazon-cloudwatch-observability" + chart = "./helm-charts/charts/amazon-cloudwatch-observability" + namespace = "amazon-cloudwatch" + create_namespace = true + wait = false + timeout = 600 + + set = [ + { name = "clusterName", value = module.eks.cluster_name }, + { name = "region", value = var.region }, + { name = "otelContainerInsights.enabled", value = "true" }, + ] + + depends_on = [ + module.eks, + null_resource.kubectl, + data.external.clone_helm_chart, + ] +} + +# --- Patch agent image --- + +resource "null_resource" "update_image" { + depends_on = [helm_release.aws_observability, null_resource.kubectl] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + sleep 30 + kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' \ + -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' + kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent-cluster-scraper --type='json' \ + -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' 2>/dev/null || true + sleep 10 + EOT + } +} + +# --- Restart pods --- + +resource "null_resource" "restart_pods" { + depends_on = [null_resource.update_image] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + kubectl -n amazon-cloudwatch rollout restart daemonset/cloudwatch-agent + kubectl -n amazon-cloudwatch rollout restart deployment/cloudwatch-agent-cluster-scraper 2>/dev/null || true + kubectl -n amazon-cloudwatch rollout status daemonset/cloudwatch-agent --timeout=120s + EOT + } +} + +# --- efaburn workload (DRA) --- +# efaburn claims 1 EFA via a ResourceClaimTemplate, so on a 2-EFA node exactly +# one device is claimed and the other stays unclaimed — exercising both the +# claimed->pod and unclaimed->no-pod correlation paths. + +resource "null_resource" "efaburn" { + depends_on = [module.eks, null_resource.kubectl, helm_release.aws_dranet] + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF' | kubectl apply -f - + apiVersion: resource.k8s.io/v1 + kind: ResourceClaimTemplate + metadata: + name: efa-claim-template + namespace: default + spec: + spec: + devices: + requests: + - name: efa-device + exactly: + deviceClassName: efa.networking.k8s.aws + count: 1 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: efaburn + namespace: default + spec: + replicas: 1 + selector: + matchLabels: + app: efaburn + template: + metadata: + labels: + app: efaburn + ci-test.example.com/pod-color: teal + spec: + nodeSelector: + ci-test.example.com/multi-efa-dra: "true" + resourceClaims: + - name: efa-device + resourceClaimTemplateName: efa-claim-template + containers: + - name: efaburn + image: ${var.efaburn_image} + resources: + claims: + - name: efa-device + requests: + memory: 8000Mi + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + EOF + EOT + } +} + +# --- Test runner --- + +resource "null_resource" "validator" { + depends_on = [ + null_resource.restart_pods, + null_resource.efaburn, + ] + + triggers = { always_run = timestamp() } + + provisioner "local-exec" { + command = <<-EOT + echo "Running OTEL Multi-EFA DRA cluster integration tests" + cd ../../../.. + + echo "Waiting 3 minutes for metrics to propagate..." + sleep 180 + + go test -tags integration -timeout 1h -v ${var.test_dir} \ + -eksClusterName=${module.eks.cluster_name} \ + -computeType=EKS \ + -eksDeploymentStrategy=DAEMON \ + -region=${var.region} + EOT + } +} diff --git a/terraform/eks/daemon/otel-multi-efa-dra/providers.tf b/terraform/eks/daemon/otel-multi-efa-dra/providers.tf new file mode 100644 index 00000000..581e0b6b --- /dev/null +++ b/terraform/eks/daemon/otel-multi-efa-dra/providers.tf @@ -0,0 +1,47 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "!= 6.22.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 3.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + } +} + +provider "aws" { + region = var.region +} + +provider "helm" { + kubernetes = { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} diff --git a/terraform/eks/daemon/otel-multi-efa-dra/variables.tf b/terraform/eks/daemon/otel-multi-efa-dra/variables.tf new file mode 100644 index 00000000..6351f17f --- /dev/null +++ b/terraform/eks/daemon/otel-multi-efa-dra/variables.tf @@ -0,0 +1,68 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +variable "region" { + type = string + default = "us-west-2" +} + +variable "test_dir" { + type = string + default = "./test/otel/multi_efa_dra" +} + +variable "cwagent_image_repo" { + type = string + default = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" +} + +variable "cwagent_image_tag" { + type = string + default = "latest" +} + +# The chart must render the DRA correlation config (dra_device_types keyed on the +# dra.net driver + the dra.net/rdmaDevice ResourceSlice attribute) and grant the +# agent ServiceAccount get/list/watch on resource.k8s.io resourceclaims and +# resourceslices. Until that lands on main, point helm_chart_repo_url/branch at +# the fork/branch that carries it (helm-charts PR #356). +variable "helm_chart_branch" { + type = string + default = "main" +} + +# Repository to clone the observability Helm chart from. Override to a fork when +# validating chart changes that are not yet merged upstream. +variable "helm_chart_repo_url" { + type = string + default = "https://github.com/aws-observability/helm-charts.git" +} + +# The awsdevicepodcorrelation processor uses the GA resource.k8s.io/v1 DRA API +# (available since Kubernetes 1.34), so the cluster must be 1.34+. +variable "k8s_version" { + type = string + default = "1.35" +} + +variable "ami_type" { + type = string + default = "AL2023_x86_64_STANDARD" +} + +variable "instance_type" { + type = string + default = "c6in.32xlarge" +} + +# dranet Helm chart version (eks/aws-dranet, from https://aws.github.io/eks-charts). +# This is the CHART version (1.0.0); the app version it ships is v1.2.0-eksbuild.2. +variable "dranet_version" { + type = string + default = "1.0.0" +} + +variable "efaburn_image" { + type = string + default = "506463145083.dkr.ecr.us-west-2.amazonaws.com/efaburn:latest" +} diff --git a/terraform/eks/daemon/otel-multi-efa-dra/vpc-endpoints.tf b/terraform/eks/daemon/otel-multi-efa-dra/vpc-endpoints.tf new file mode 100644 index 00000000..6a136138 --- /dev/null +++ b/terraform/eks/daemon/otel-multi-efa-dra/vpc-endpoints.tf @@ -0,0 +1,82 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +# VPC Endpoints for private subnet access +resource "aws_vpc_endpoint" "ecr_dkr" { + vpc_id = aws_vpc.efa_test_vpc.id + service_name = "com.amazonaws.${var.region}.ecr.dkr" + vpc_endpoint_type = "Interface" + subnet_ids = aws_subnet.efa_test_private_subnet[*].id + security_group_ids = [aws_security_group.vpc_endpoints.id] + private_dns_enabled = true + + tags = { + Name = "efa-test-ecr-dkr-${module.common.testing_id}" + Owner = "default" + } +} + +resource "aws_vpc_endpoint" "ecr_api" { + vpc_id = aws_vpc.efa_test_vpc.id + service_name = "com.amazonaws.${var.region}.ecr.api" + vpc_endpoint_type = "Interface" + subnet_ids = aws_subnet.efa_test_private_subnet[*].id + security_group_ids = [aws_security_group.vpc_endpoints.id] + private_dns_enabled = true + + tags = { + Name = "efa-test-ecr-api-${module.common.testing_id}" + Owner = "default" + } +} + +resource "aws_vpc_endpoint" "s3" { + vpc_id = aws_vpc.efa_test_vpc.id + service_name = "com.amazonaws.${var.region}.s3" + vpc_endpoint_type = "Gateway" + route_table_ids = [aws_route_table.efa_test_private_rt.id] + + tags = { + Name = "efa-test-s3-${module.common.testing_id}" + Owner = "default" + } +} + +resource "aws_vpc_endpoint" "eks" { + vpc_id = aws_vpc.efa_test_vpc.id + service_name = "com.amazonaws.${var.region}.eks" + vpc_endpoint_type = "Interface" + subnet_ids = aws_subnet.efa_test_private_subnet[*].id + security_group_ids = [aws_security_group.vpc_endpoints.id] + private_dns_enabled = true + + tags = { + Name = "efa-test-eks-${module.common.testing_id}" + Owner = "default" + } +} + +# Security group for VPC endpoints +resource "aws_security_group" "vpc_endpoints" { + name_prefix = "efa-test-vpc-endpoints-${module.common.testing_id}" + vpc_id = aws_vpc.efa_test_vpc.id + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = [aws_vpc.efa_test_vpc.cidr_block] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "efa-test-vpc-endpoints-sg-${module.common.testing_id}" + Owner = "default" + } +} diff --git a/terraform/eks/daemon/otel-multi-efa-dra/vpc.tf b/terraform/eks/daemon/otel-multi-efa-dra/vpc.tf new file mode 100644 index 00000000..1db6495f --- /dev/null +++ b/terraform/eks/daemon/otel-multi-efa-dra/vpc.tf @@ -0,0 +1,152 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +# VPC for EFA integration test +resource "aws_vpc" "efa_test_vpc" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + Name = "efa-test-vpc-${module.common.testing_id}" + Owner = "default" + } +} + +# Internet Gateway +resource "aws_internet_gateway" "efa_test_igw" { + vpc_id = aws_vpc.efa_test_vpc.id + + tags = { + Name = "efa-test-igw-${module.common.testing_id}" + Owner = "default" + } +} + +# Public Subnets +resource "aws_subnet" "efa_test_public_subnet" { + count = 2 + + vpc_id = aws_vpc.efa_test_vpc.id + cidr_block = "10.0.${count.index + 1}.0/24" + availability_zone = data.aws_availability_zones.available.names[count.index] + map_public_ip_on_launch = true + + tags = { + Name = "efa-test-public-subnet-${count.index + 1}-${module.common.testing_id}" + Owner = "default" + } +} + +# Route Table for Public Subnets +resource "aws_route_table" "efa_test_public_rt" { + vpc_id = aws_vpc.efa_test_vpc.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.efa_test_igw.id + } + + tags = { + Name = "efa-test-public-rt-${module.common.testing_id}" + Owner = "default" + } +} + +# Associate Route Table with Public Subnets +resource "aws_route_table_association" "efa_test_public_rta" { + count = length(aws_subnet.efa_test_public_subnet) + + subnet_id = aws_subnet.efa_test_public_subnet[count.index].id + route_table_id = aws_route_table.efa_test_public_rt.id +} + +# Security Group +resource "aws_security_group" "efa_test_sg" { + name_prefix = "efa-test-sg-${module.common.testing_id}" + vpc_id = aws_vpc.efa_test_vpc.id + + ingress { + from_port = 0 + to_port = 65535 + protocol = "tcp" + self = true + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "efa-test-sg-${module.common.testing_id}" + Owner = "default" + } +} + +# Private Subnets +resource "aws_subnet" "efa_test_private_subnet" { + count = 2 + + vpc_id = aws_vpc.efa_test_vpc.id + cidr_block = "10.0.${count.index + 10}.0/24" + availability_zone = data.aws_availability_zones.available.names[count.index] + + tags = { + Name = "efa-test-private-subnet-${count.index + 1}-${module.common.testing_id}" + Owner = "default" + } +} + +# Elastic IP for NAT Gateway +resource "aws_eip" "efa_test_nat_eip" { + domain = "vpc" + + tags = { + Name = "efa-test-nat-eip-${module.common.testing_id}" + Owner = "default" + } +} + +# NAT Gateway +resource "aws_nat_gateway" "efa_test_nat" { + allocation_id = aws_eip.efa_test_nat_eip.id + subnet_id = aws_subnet.efa_test_public_subnet[0].id + + tags = { + Name = "efa-test-nat-${module.common.testing_id}" + Owner = "default" + } + + depends_on = [aws_internet_gateway.efa_test_igw] +} + +# Route Table for Private Subnets +resource "aws_route_table" "efa_test_private_rt" { + vpc_id = aws_vpc.efa_test_vpc.id + + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.efa_test_nat.id + } + + tags = { + Name = "efa-test-private-rt-${module.common.testing_id}" + Owner = "default" + } +} + +# Associate Route Table with Private Subnets +resource "aws_route_table_association" "efa_test_private_rta" { + count = length(aws_subnet.efa_test_private_subnet) + + subnet_id = aws_subnet.efa_test_private_subnet[count.index].id + route_table_id = aws_route_table.efa_test_private_rt.id +} + +# Data source for availability zones +data "aws_availability_zones" "available" { + state = "available" +} diff --git a/terraform/eks/daemon/otel-neuron-dra/main.tf b/terraform/eks/daemon/otel-neuron-dra/main.tf new file mode 100644 index 00000000..fe3c2325 --- /dev/null +++ b/terraform/eks/daemon/otel-neuron-dra/main.tf @@ -0,0 +1,419 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +# Neuron DRA-path integration test cluster. Mirrors otel-neuron but exposes Neuron +# devices via the AWS Neuron DRA driver (DeviceClass neuron.aws.com) instead of the +# device plugin, and the burn workload claims a device via a ResourceClaimTemplate. +# One Trainium node (trn1.2xlarge = 1 device x 2 cores) proves per-device/per-core +# DRA correlation: the claimed device's 2 cores attribute to the burn pod, and to +# no other pod. Trainium (not Inferentia): the Neuron DRA driver image 1.2.0 supports +# Trainium only and rejects inf1/inf2 at device discovery ("unsupported instance +# type"); trn1.2xlarge is the smallest/most-available Trainium instance. + +module "common" { + source = "../../../common" + cwagent_image_repo = var.cwagent_image_repo + cwagent_image_tag = var.cwagent_image_tag +} + +module "basic_components" { + source = "../../../basic_components" + region = var.region +} + +locals { + aws_eks = "aws eks --region ${var.region}" +} + +resource "aws_eks_cluster" "this" { + name = "cwagent-eks-integ-${module.common.testing_id}" + role_arn = module.basic_components.role_arn + version = var.k8s_version + vpc_config { + subnet_ids = module.basic_components.public_subnet_ids + security_group_ids = [module.basic_components.security_group] + } +} + +# --- IAM --- + +resource "aws_iam_role" "node_role" { + name = "cwagent-otel-neuron-dra-Worker-Role-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEKSWorkerNodePolicy" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEKS_CNI_Policy" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEC2ContainerRegistryReadOnly" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role" "pod_identity_role" { + name = "cwagent-otel-neuron-dra-pod-identity-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) +} + +resource "aws_iam_role_policy_attachment" "pod_identity_CloudWatchAgentServerPolicy" { + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" + role = aws_iam_role.pod_identity_role.name +} + +# --- Node Groups --- + +resource "aws_eks_node_group" "standard" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "standard-${module.common.testing_id}" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = module.basic_components.public_subnet_ids + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + ami_type = "AL2023_x86_64_STANDARD" + capacity_type = "ON_DEMAND" + disk_size = 20 + instance_types = ["t3.medium"] + + depends_on = [ + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, + ] +} + +# Neuron (Trainium) node (trn1.2xlarge). No taint: the node is dedicated by +# being the only Neuron node, and the DRA driver DaemonSet + burn pod land here via +# scheduling/nodeSelector. Avoiding a taint keeps the Neuron DRA driver DaemonSet +# (whose default tolerations are not assumed) schedulable. +# Pinned to usw2-az4: trn1.2xlarge is offered only in usw2-az1/usw2-az4 in us-west-2. +data "aws_subnet" "az" { + for_each = toset(module.basic_components.public_subnet_ids) + id = each.value +} + +locals { + neuron_az_id = "usw2-az4" + neuron_subnets = [for s in data.aws_subnet.az : s.id if s.availability_zone_id == local.neuron_az_id] +} + +resource "aws_eks_node_group" "neuron_multi" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "neuron-multi-${module.common.testing_id}" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = local.neuron_subnets + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + ami_type = var.ami_type + capacity_type = "ON_DEMAND" + disk_size = 100 + instance_types = [var.instance_type] + + labels = { + "aws.amazon.com/neuron.present" = "true" + "ci-test.example.com/node-color" = "purple" + } + + depends_on = [ + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, + ] +} + +# --- EKS Addon: Pod Identity agent --- + +resource "aws_eks_addon" "pod_identity_agent" { + depends_on = [aws_eks_node_group.standard] + cluster_name = aws_eks_cluster.this.name + addon_name = "eks-pod-identity-agent" +} + +# --- Update kubeconfig --- + +resource "null_resource" "kubectl" { + depends_on = [aws_eks_cluster.this, aws_eks_node_group.standard, aws_eks_node_group.neuron_multi] + provisioner "local-exec" { + command = "${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name}" + } +} + +# --- Neuron DRA driver (Helm) — replaces the device plugin --- +# DeviceClass neuron.aws.com, driver neuron.aws.com. devicePlugin disabled so it +# does not compete with the DRA driver for the same devices. + +resource "helm_release" "neuron_dra_driver" { + depends_on = [aws_eks_node_group.neuron_multi, null_resource.kubectl] + + name = "neuron-helm-chart" + repository = "oci://public.ecr.aws/neuron" + chart = "neuron-helm-chart" + # The chart renders its own Namespace object for neuron-dra-driver (via + # draDriver.namespaceOverride, no disable toggle) AND places the driver + # DaemonSet/SA/RBAC there. So we install the release into the pre-existing + # kube-system namespace (Helm needs the release namespace to exist to store the + # release secret) with create_namespace=false, and let the chart create and own + # neuron-dra-driver. Installing into neuron-dra-driver directly deadlocks: with + # create_namespace=true the provider's namespace collides with the chart's + # ("already exists"); with false Helm fails "namespace not found". + namespace = "kube-system" + create_namespace = false + version = var.neuron_helm_chart_version != "" ? var.neuron_helm_chart_version : null + + set = [ + { name = "devicePlugin.enabled", value = "false" }, + { name = "npd.enabled", value = "false" }, + { name = "scheduler.enabled", value = "false" }, + { name = "draDriver.enabled", value = "true" }, + ] +} + +# --- neuron-burn-dra Deployment: claims 1 whole Neuron device via DRA --- + +resource "null_resource" "neuron_burn_dra" { + depends_on = [helm_release.neuron_dra_driver, null_resource.kubectl] + # Re-apply the manifest (kubectl apply is idempotent) if the instance type + # changes, so the burn pod's node-type nodeSelector tracks var.instance_type. + triggers = { + instance_type = var.instance_type + cluster = aws_eks_cluster.this.name + } + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF' | kubectl apply -f - + apiVersion: resource.k8s.io/v1 + kind: ResourceClaimTemplate + metadata: + name: neuron-1 + namespace: default + spec: + spec: + devices: + requests: + - name: neuron + exactly: + deviceClassName: neuron.aws.com + count: 1 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: neuron-burn-dra + namespace: default + spec: + replicas: 1 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 600 + selector: + matchLabels: + app: neuron-burn-dra + template: + metadata: + labels: + app: neuron-burn-dra + neuron-test: "true" + ci-test.example.com/pod-color: violet + spec: + nodeSelector: + node.kubernetes.io/instance-type: ${var.instance_type} + resourceClaims: + - name: neuron + resourceClaimTemplateName: neuron-1 + containers: + - name: neuron-burn + image: public.ecr.aws/neuron/pytorch-inference-neuronx:2.1.2-neuronx-py310-sdk2.20.2-ubuntu20.04 + command: ["python3", "-c"] + args: + - | + import torch, torch_neuronx, time + print("Compiling neuron trace (this takes a minute)...") + x = torch.randn(256, 256) + model = torch.nn.Linear(256, 256, bias=False) + traced = torch_neuronx.trace(model, x) + print("Trace compiled. Starting burn loop...") + iteration = 0 + while True: + start = time.time() + for _ in range(1000): + _ = traced(x) + elapsed = time.time() - start + iteration += 1 + print(f"Iteration {iteration}: 1000 inferences in {elapsed:.2f}s") + resources: + claims: + - name: neuron + requests: + cpu: "1" + memory: 4Gi + EOF + EOT + } +} + +# --- Helm chart install (observability) --- + +data "external" "clone_helm_chart" { + program = ["bash", "-c", <<-EOT + rm -rf ./helm-charts + git clone -b ${var.helm_chart_branch} ${var.helm_chart_repo_url} ./helm-charts + echo '{"status":"ready"}' + EOT + ] +} + +resource "helm_release" "aws_observability" { + name = "amazon-cloudwatch-observability" + chart = "./helm-charts/charts/amazon-cloudwatch-observability" + namespace = "amazon-cloudwatch" + create_namespace = true + wait = false + timeout = 600 + + set = [ + { name = "clusterName", value = aws_eks_cluster.this.name }, + { name = "region", value = var.region }, + { name = "otelContainerInsights.enabled", value = "true" }, + ] + + depends_on = [ + aws_eks_addon.pod_identity_agent, + null_resource.kubectl, + data.external.clone_helm_chart, + ] +} + +# --- Pod Identity association --- + +resource "aws_eks_pod_identity_association" "cloudwatch_agent" { + depends_on = [helm_release.aws_observability] + cluster_name = aws_eks_cluster.this.name + namespace = "amazon-cloudwatch" + service_account = "cloudwatch-agent" + role_arn = aws_iam_role.pod_identity_role.arn +} + +# --- Patch agent image --- + +resource "null_resource" "update_image" { + depends_on = [helm_release.aws_observability, null_resource.kubectl] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + sleep 30 + kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' \ + -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' + kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent-cluster-scraper --type='json' \ + -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' 2>/dev/null || true + sleep 10 + EOT + } +} + +# --- Restart pods --- + +resource "null_resource" "restart_pods" { + depends_on = [aws_eks_pod_identity_association.cloudwatch_agent, null_resource.update_image] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + kubectl -n amazon-cloudwatch rollout restart daemonset/cloudwatch-agent + kubectl -n amazon-cloudwatch rollout restart deployment/cloudwatch-agent-cluster-scraper 2>/dev/null || true + kubectl -n amazon-cloudwatch rollout status daemonset/cloudwatch-agent --timeout=120s + EOT + } +} + +# --- Wait for neuron-monitor pods --- + +resource "null_resource" "wait_neuron_monitor" { + depends_on = [null_resource.restart_pods, null_resource.neuron_burn_dra] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + echo "Waiting for neuron-monitor pods to be ready..." + for i in $(seq 1 30); do + READY=$(kubectl -n amazon-cloudwatch get pods -l app.kubernetes.io/name=neuron-monitor --no-headers 2>/dev/null | grep -c "Running" || true) + if [ "$READY" -ge 1 ]; then + echo "neuron-monitor ready ($READY running)" + break + fi + echo "Attempt $i: $READY neuron-monitor pods running, waiting..." + sleep 10 + done + EOT + } +} + +# --- Test runner --- + +resource "null_resource" "validator" { + depends_on = [null_resource.wait_neuron_monitor] + triggers = { always_run = timestamp() } + provisioner "local-exec" { + command = <<-EOT + echo "Running OTEL Neuron DRA cluster integration tests" + cd ../../../.. + + echo "Waiting for Neuron runtime to initialize (image pull + trace compile)..." + READY=0 + for i in $(seq 1 90); do + READY=$(kubectl logs -n default -l app=neuron-burn-dra --tail=5 2>/dev/null | grep -c "^Iteration " || true) + if [ "$READY" -gt 0 ]; then + echo "Neuron burn loop active (after $((i*10))s)" + break + fi + if [ $((i % 6)) -eq 0 ]; then + echo "--- Attempt $i ($((i*10))s): iter_lines=$READY ---" + kubectl get pods -n default -l app=neuron-burn-dra -o wide 2>&1 | head || true + fi + sleep 10 + done + if [ "$READY" -eq 0 ]; then + echo "ERROR: neuron-burn-dra loop not active after 15 minutes" + kubectl get pods -n default -l app=neuron-burn-dra -o wide 2>&1 || true + kubectl describe pod -n default -l app=neuron-burn-dra 2>&1 | tail -40 || true + kubectl get resourceclaims -n default 2>&1 || true + exit 1 + fi + + echo "Waiting 6 minutes for metrics to propagate (Zeus 5-min staleness window)..." + sleep 360 + + go test -tags integration -timeout 1h -v ${var.test_dir} \ + -eksClusterName=${aws_eks_cluster.this.name} \ + -computeType=EKS \ + -eksDeploymentStrategy=DAEMON \ + -region=${var.region} + EOT + } +} diff --git a/terraform/eks/daemon/otel-neuron-dra/providers.tf b/terraform/eks/daemon/otel-neuron-dra/providers.tf new file mode 100644 index 00000000..a562a7f1 --- /dev/null +++ b/terraform/eks/daemon/otel-neuron-dra/providers.tf @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +provider "aws" { + region = var.region +} + +provider "kubernetes" { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } +} + +provider "helm" { + kubernetes = { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } + } +} diff --git a/terraform/eks/daemon/otel-neuron-dra/variables.tf b/terraform/eks/daemon/otel-neuron-dra/variables.tf new file mode 100644 index 00000000..bc7bdad1 --- /dev/null +++ b/terraform/eks/daemon/otel-neuron-dra/variables.tf @@ -0,0 +1,66 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +variable "region" { + type = string + default = "us-west-2" +} + +variable "test_dir" { + type = string + default = "./test/otel/neuron_dra" +} + +variable "cwagent_image_repo" { + type = string + default = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" +} + +variable "cwagent_image_tag" { + type = string + default = "latest" +} + +# The chart must render the DRA correlation config (dra_device_types incl. the +# neuron-dra entry keyed on the neuron.aws.com driver) and grant the agent +# ServiceAccount resource.k8s.io RBAC. Override to a fork/branch until that lands +# on main (helm-charts PR #356). +variable "helm_chart_branch" { + type = string + default = "main" +} + +variable "helm_chart_repo_url" { + type = string + default = "https://github.com/aws-observability/helm-charts.git" +} + +# The awsdevicepodcorrelation processor uses the GA resource.k8s.io/v1 DRA API +# (available since Kubernetes 1.34), so the cluster must be 1.34+. +variable "k8s_version" { + type = string + default = "1.35" +} + +variable "ami_type" { + type = string + default = "AL2023_x86_64_NEURON" +} + +# Neuron (Trainium) node: trn1.2xlarge = 1 device × 2 cores. +# Must be a Trainium type: the Neuron DRA driver (driver image 1.2.0) supports +# Trainium only and rejects Inferentia (inf1/inf2) at device discovery. trn1.2xlarge +# is the smallest/most-available Trainium instance, enough to exercise DRA +# per-device correlation (1 claimed device -> both cores -> burn pod). +variable "instance_type" { + type = string + default = "trn1.2xlarge" +} + +# Neuron DRA driver Helm chart (OCI). draDriver.enabled=true installs the DRA +# driver (DeviceClass neuron.aws.com); devicePlugin must be disabled (the two +# cannot coexist on a node). +variable "neuron_helm_chart_version" { + type = string + default = "" +} diff --git a/test/otel/gpu_dra/gpu_dra_test.go b/test/otel/gpu_dra/gpu_dra_test.go new file mode 100644 index 00000000..5fd88859 --- /dev/null +++ b/test/otel/gpu_dra/gpu_dra_test.go @@ -0,0 +1,170 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +// Package gpu_dra validates per-device GPU pod correlation on the Dynamic Resource +// Allocation (DRA) path — the counterpart of the device-plugin `gpu` package's +// multi_gpu tests. GPUs are allocated via the NVIDIA DRA driver (DeviceClass +// gpu.nvidia.com, driver gpu.nvidia.com) through a ResourceClaimTemplate instead of +// the nvidia.com/gpu device-plugin resource. The emitted DCGM metrics are identical, +// so the assertions mirror the device-plugin multi_gpu_test. +// +// Cluster topology: +// - 1x g4dn.12xlarge = 4 T4 GPUs +// - GPUs exposed via the NVIDIA DRA driver, not the device plugin +// - A burn workload (gpu-burn-dra) claims 1 GPU via DRA +package gpu_dra + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + multiGpuInstanceType = "g4dn.12xlarge" + expectedMultiGPUCount = 4 + + // burnPodPrefix is the DRA burn workload that claims 1 GPU via a ResourceClaimTemplate. + burnPodPrefix = "gpu-burn-dra" + expectedClaimedGPUs = 1 + expectedUncorrelated = expectedMultiGPUCount - expectedClaimedGPUs +) + +// TestGPUDRADeviceCount validates the node exposes the expected number of GPUs. +func TestGPUDRADeviceCount(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") + require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") + require.NotEmpty(t, results, "DCGM_FI_DEV_GPU_UTIL not available") + + multi := filterByHostType(results, multiGpuInstanceType) + require.True(t, len(multi) > 0, "No DCGM_FI_DEV_GPU_UTIL results from %s node", multiGpuInstanceType) + + gpus := uniqueDatapointValues(multi, "gpu") + require.Equal(t, expectedMultiGPUCount, len(gpus), + "Expected %d distinct gpu on %s, got %d: %v", expectedMultiGPUCount, multiGpuInstanceType, len(gpus), gpus) +} + +// TestGPUDRAConsecutiveIndices validates GPU indices are 0..N-1. +func TestGPUDRAConsecutiveIndices(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") + require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") + multi := filterByHostType(results, multiGpuInstanceType) + require.True(t, len(multi) > 0, "No results from %s node", multiGpuInstanceType) + + gpus := uniqueDatapointValues(multi, "gpu") + require.Equal(t, expectedMultiGPUCount, len(gpus), "Expected %d GPU indices, got %d: %v", expectedMultiGPUCount, len(gpus), gpus) + for i := 0; i < expectedMultiGPUCount; i++ { + expected := strconv.Itoa(i) + found := false + for _, g := range gpus { + if g == expected { + found = true + break + } + } + require.True(t, found, "Expected GPU index %d, got indices: %v", i, gpus) + } +} + +// TestGPUDRAAllMetricsPerDevice validates every DCGM metric reports for all 4 GPUs. +func TestGPUDRAAllMetricsPerDevice(t *testing.T) { + t.Parallel() + for _, metricName := range dcgmMetricNamesList { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + multi := filterByHostType(results, multiGpuInstanceType) + require.True(t, len(multi) > 0, "No %s results from %s node", metricName, multiGpuInstanceType) + + gpus := uniqueDatapointValues(multi, "gpu") + require.Equal(t, expectedMultiGPUCount, len(gpus), + "%s: expected %d GPUs, got %d: %v", metricName, expectedMultiGPUCount, len(gpus), gpus) + }) + } +} + +// TestGPUDRAClaimedVsUnclaimedCorrelation is the DRA-path per-device correlation +// guard. On a 4-GPU node, the burn pod claims exactly 1 GPU via DRA, so exactly 1 +// GPU correlates to that pod and the remaining 3 carry NO pod. This fails if DRA +// correlation collapses GPUs onto one pod, over-correlates unclaimed GPUs, or maps +// a GPU to the wrong pod. +func TestGPUDRAClaimedVsUnclaimedCorrelation(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "DCGM_FI_DEV_GPU_UTIL") + require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL") + multi := filterByHostType(results, multiGpuInstanceType) + require.NotEmpty(t, multi, "No DCGM_FI_DEV_GPU_UTIL results from %s node", multiGpuInstanceType) + + // For each GPU, collect the distinct pods it is attributed to (empty = unclaimed). + gpuPods := make(map[string]map[string]struct{}) + for _, r := range multi { + r := r + gpu := r.Labels.Datapoint["gpu"] + if gpu == "" { + continue + } + if gpuPods[gpu] == nil { + gpuPods[gpu] = make(map[string]struct{}) + } + if pod := r.Labels.Resource["k8s.pod.name"]; pod != "" { + gpuPods[gpu][pod] = struct{}{} + } + } + require.Len(t, gpuPods, expectedMultiGPUCount, + "expected %d GPUs on the node, got %d", expectedMultiGPUCount, len(gpuPods)) + + var claimed, unclaimed []string + for gpu, pods := range gpuPods { + switch len(pods) { + case 0: + unclaimed = append(unclaimed, gpu) + case 1: + var pod string + for p := range pods { + pod = p + } + require.True(t, strings.HasPrefix(pod, burnPodPrefix), + "GPU %s correlated to unexpected pod %q (expected %s*)", gpu, pod, burnPodPrefix) + claimed = append(claimed, gpu) + default: + t.Errorf("GPU %s correlated to multiple pods", gpu) + } + } + + require.Len(t, claimed, expectedClaimedGPUs, + "expected %d claimed GPU correlated to %s*, got %d: %v "+ + "(collapse over-correlates unclaimed GPUs onto a pod)", + expectedClaimedGPUs, burnPodPrefix, len(claimed), claimed) + require.Len(t, unclaimed, expectedUncorrelated, + "expected %d uncorrelated GPUs with no pod, got %d: %v", + expectedUncorrelated, len(unclaimed), unclaimed) +} + +// TestGPUDRABurnPodLabels validates the correlated GPU's pod labels via a targeted +// PromQL query (avoids stale series in the shared OTLP store). +func TestGPUDRABurnPodLabels(t *testing.T) { + t.Parallel() + ctx := context.Background() + promql := fmt.Sprintf( + `DCGM_FI_DEV_GPU_UTIL{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"%s.*"}`, + escapePromQL(cfg.ClusterName), burnPodPrefix) + burn, err := client.Query(ctx, promql) + require.NoError(t, err, "querying DCGM_FI_DEV_GPU_UTIL for %s", burnPodPrefix) + require.Equal(t, expectedClaimedGPUs, len(burn), + "Expected %d GPU correlated to %s, got %d", expectedClaimedGPUs, burnPodPrefix, len(burn)) + + for _, r := range burn { + r := r + require.Equal(t, "default", r.Labels.Resource["k8s.namespace.name"], "%s namespace", burnPodPrefix) + } +} diff --git a/test/otel/gpu_dra/helpers_test.go b/test/otel/gpu_dra/helpers_test.go new file mode 100644 index 00000000..80774242 --- /dev/null +++ b/test/otel/gpu_dra/helpers_test.go @@ -0,0 +1,46 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package gpu_dra + +import ( + "sort" + "strings" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// escapePromQL escapes a string value for safe use inside PromQL label matches. +func escapePromQL(s string) string { + return strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) +} + +// filterByHostType returns only results whose resource host.type matches. +func filterByHostType(results []otelmetrics.MetricResult, instanceType string) []otelmetrics.MetricResult { + var out []otelmetrics.MetricResult + for _, r := range results { + if r.Labels.Resource["host.type"] == instanceType { + out = append(out, r) + } + } + return out +} + +// uniqueDatapointValues returns the sorted unique non-empty values of a +// datapoint-level attribute across all results. +func uniqueDatapointValues(results []otelmetrics.MetricResult, attr string) []string { + seen := make(map[string]struct{}) + for _, r := range results { + if v, ok := r.Labels.Datapoint[attr]; ok && v != "" { + seen[v] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for v := range seen { + out = append(out, v) + } + sort.Strings(out) + return out +} diff --git a/test/otel/gpu_dra/metrics_test.go b/test/otel/gpu_dra/metrics_test.go new file mode 100644 index 00000000..24e6de09 --- /dev/null +++ b/test/otel/gpu_dra/metrics_test.go @@ -0,0 +1,31 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package gpu_dra + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +// The DRA cluster has a single multi-GPU node (g4dn.12xlarge, 4 GPUs). +var clusterHostTypes = []string{multiGpuInstanceType} + +// DCGM metric definitions (same source names as the device-plugin gpu package). +var dcgmMetrics = []otelmetrics.MetricDefinition{ + {Name: "DCGM_FI_DEV_GPU_UTIL", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "%"}, + {Name: "DCGM_FI_DEV_MEM_COPY_UTIL", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "%"}, + {Name: "DCGM_FI_DEV_FB_USED", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "MiBy"}, + {Name: "DCGM_FI_DEV_GPU_TEMP", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "Cel"}, + {Name: "DCGM_FI_DEV_POWER_USAGE", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "W"}, + {Name: "DCGM_FI_DEV_FB_FREE", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "MiBy"}, +} + +func metricNames(defs []otelmetrics.MetricDefinition) []string { + names := make([]string, len(defs)) + for i, d := range defs { + names[i] = d.Name + } + return names +} + +var dcgmMetricNamesList = metricNames(dcgmMetrics) diff --git a/test/otel/gpu_dra/setup_test.go b/test/otel/gpu_dra/setup_test.go new file mode 100644 index 00000000..9aeb334f --- /dev/null +++ b/test/otel/gpu_dra/setup_test.go @@ -0,0 +1,95 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package gpu_dra + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +var ( + cfg otelmetrics.TestConfig + client *otelmetrics.OtelMetricsClient + queryCache *otelmetrics.QueryCache +) + +func TestMain(m *testing.M) { + environment.RegisterEnvironmentMetaDataFlags() + flag.Parse() + env := environment.GetEnvironmentMetaData() + + region := env.Region + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + fmt.Fprintf(os.Stderr, "Region not set\n") + os.Exit(1) + } + + clusterName := env.EKSClusterName + if clusterName == "" { + clusterName = os.Getenv("CLUSTER_NAME") + } + if clusterName == "" { + fmt.Fprintf(os.Stderr, "Cluster name not set\n") + os.Exit(1) + } + + ctx := context.Background() + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + fmt.Fprintf(os.Stderr, "AWS config error: %v\n", err) + os.Exit(1) + } + stsClient := sts.NewFromConfig(awsCfg) + identity, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + fmt.Fprintf(os.Stderr, "STS GetCallerIdentity error: %v\n", err) + os.Exit(1) + } + + cfg = otelmetrics.TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + ClusterName: clusterName, + AccountID: *identity.Account, + SigningService: "monitoring", + } + + client, err = otelmetrics.NewClient(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Client error: %v\n", err) + os.Exit(1) + } + + hostMappings := []otelmetrics.SourceHostMapping{ + {Source: otelmetrics.SourceDCGM, HostTypes: clusterHostTypes}, + } + + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, + otelmetrics.SourceMapping{Source: otelmetrics.SourceDCGM, Metrics: dcgmMetrics}, + ) + + queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithHostTypes(clusterHostTypes), + otelmetrics.WithSourceRegistry(registry), + ) + + os.Exit(m.Run()) +} diff --git a/test/otel/multi_efa_dra/helpers_test.go b/test/otel/multi_efa_dra/helpers_test.go new file mode 100644 index 00000000..15748188 --- /dev/null +++ b/test/otel/multi_efa_dra/helpers_test.go @@ -0,0 +1,71 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package multi_efa_dra + +import ( + "sort" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// getAnyValue returns the attribute value from either resource or datapoint scope, +// preferring resource scope if set in both. +func getAnyValue(r otelmetrics.MetricResult, attr string) string { + if v, ok := r.Labels.Resource[attr]; ok && v != "" { + return v + } + return r.Labels.Datapoint[attr] +} + +// filterByNodeLabel returns results where the given resource attribute equals the given value. +func filterByNodeLabel(results []otelmetrics.MetricResult, labelKey, labelValue string) []otelmetrics.MetricResult { + var out []otelmetrics.MetricResult + for _, r := range results { + if r.Labels.Resource[labelKey] == labelValue { + out = append(out, r) + } + } + return out +} + +// deviceKeys returns the sorted device names of a device->pods map (for messages). +func deviceKeys(m map[string]map[string]struct{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// setKeys returns the sorted keys of a string set (for messages). +func setKeys(m map[string]struct{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// uniqueAnyValues returns the sorted unique non-empty values of an attribute +// found in either resource or datapoint scope across all results. +func uniqueAnyValues(results []otelmetrics.MetricResult, attr string) []string { + seen := make(map[string]struct{}) + for _, r := range results { + if v, ok := r.Labels.Resource[attr]; ok && v != "" { + seen[v] = struct{}{} + } else if v, ok := r.Labels.Datapoint[attr]; ok && v != "" { + seen[v] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for v := range seen { + out = append(out, v) + } + sort.Strings(out) + return out +} diff --git a/test/otel/multi_efa_dra/metrics_test.go b/test/otel/multi_efa_dra/metrics_test.go new file mode 100644 index 00000000..7aa4afeb --- /dev/null +++ b/test/otel/multi_efa_dra/metrics_test.go @@ -0,0 +1,25 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package multi_efa_dra + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +var efaMetrics = []otelmetrics.MetricDefinition{ + {Name: "efa_rx_bytes", MetricType: "counter", Scope: otelmetrics.ScopePod, Unit: "By"}, + {Name: "efa_tx_bytes", MetricType: "counter", Scope: otelmetrics.ScopePod, Unit: "By"}, + {Name: "efa_rx_dropped", MetricType: "counter", Scope: otelmetrics.ScopePod}, + {Name: "efa_rdma_read_bytes", MetricType: "counter", Scope: otelmetrics.ScopePod, Unit: "By"}, +} + +var efaMetricNamesList = metricNames(efaMetrics) + +func metricNames(defs []otelmetrics.MetricDefinition) []string { + names := make([]string, len(defs)) + for i, d := range defs { + names[i] = d.Name + } + return names +} diff --git a/test/otel/multi_efa_dra/multi_efa_dra_test.go b/test/otel/multi_efa_dra/multi_efa_dra_test.go new file mode 100644 index 00000000..f24027f3 --- /dev/null +++ b/test/otel/multi_efa_dra/multi_efa_dra_test.go @@ -0,0 +1,233 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +// Package multi_efa_dra validates per-device EFA metric correlation on the +// Dynamic Resource Allocation (DRA) path — the same behavior the multi_efa +// package validates for the EFA device-plugin path, but with EFA devices +// allocated via DRA (the dranet driver, driver name "dra.net") through a +// ResourceClaimTemplate instead of the vpc.amazonaws.com/efa device-plugin +// resource. +// +// This exercises the awsdevicepodcorrelation processor's DRA path end to end: +// the processor watches Pods/ResourceClaims/ResourceSlices via the K8s API and +// bridges the DRA device identity (a PCI name) to the EFA metric label +// (rdmapXsY) via the ResourceSlice attribute dra.net/rdmaDevice. The emitted +// metrics are identical to the device-plugin path, so the assertions mirror the +// multi_efa package. +// +// Cluster topology: +// - 1x c6in.32xlarge with 2 EFA interfaces (1 per network card) +// - EFA exposed via DRA (dranet), not the EFA device plugin +// - Node label: ci-test.example.com/multi-efa-dra=true +package multi_efa_dra + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + expectedMultiEFACount = 2 + multiEfaDraNodeLabel = "k8s.node.label.ci-test.example.com/multi-efa-dra" +) + +func TestMultiEFADeviceCount(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "efa_rx_bytes") + require.NoError(t, err, "querying efa_rx_bytes") + require.NotEmpty(t, results, "efa_rx_bytes not available") + + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + require.True(t, len(multi) > 0, + "No efa_rx_bytes results from multi-EFA DRA node (label %s)", multiEfaDraNodeLabel) + + devices := uniqueAnyValues(multi, "aws.efa.device") + require.Equal(t, expectedMultiEFACount, len(devices), + "Expected %d distinct aws.efa.device, got %d: %v", + expectedMultiEFACount, len(devices), devices) +} + +func TestMultiEFAUniqueENIs(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "efa_rx_bytes") + require.NoError(t, err, "querying efa_rx_bytes") + require.NotEmpty(t, results, "efa_rx_bytes not available") + + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + require.True(t, len(multi) > 0, "No results from multi-EFA DRA node") + + enis := uniqueAnyValues(multi, "aws.efa.eni.id") + devices := uniqueAnyValues(multi, "aws.efa.device") + require.Equal(t, len(devices), len(enis), + "EFA device count (%d) != ENI count (%d) — duplicate ENIs?", + len(devices), len(enis)) + for _, eni := range enis { + require.True(t, strings.HasPrefix(eni, "eni-"), + "aws.efa.eni.id should start with 'eni-', got '%s'", eni) + } +} + +func TestMultiEFADeviceNamesAreRDMA(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "efa_rx_bytes") + require.NoError(t, err, "querying efa_rx_bytes") + require.NotEmpty(t, results, "efa_rx_bytes not available") + + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + require.True(t, len(multi) > 0, "No results from multi-EFA DRA node") + + for _, r := range multi { + dev := getAnyValue(r, "aws.efa.device") + require.NotEmpty(t, dev, "aws.efa.device is empty") + } +} + +func TestMultiEFAAllMetricsPerDevice(t *testing.T) { + t.Parallel() + for _, metricName := range efaMetricNamesList { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + require.True(t, len(multi) > 0, "No %s results from multi-EFA DRA node", metricName) + + devices := uniqueAnyValues(multi, "aws.efa.device") + require.Equal(t, expectedMultiEFACount, len(devices), + "%s: expected %d EFA devices, got %d: %v", + metricName, expectedMultiEFACount, len(devices), devices) + }) + } +} + +func TestMultiEFAPortPresent(t *testing.T) { + t.Parallel() + for _, metricName := range efaMetricNamesList { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + require.True(t, len(multi) > 0, "No %s results from multi-EFA DRA node", metricName) + + for _, r := range multi { + port := getAnyValue(r, "aws.efa.port") + require.NotEmpty(t, port, + "%s missing aws.efa.port (device: %s)", + metricName, getAnyValue(r, "aws.efa.device")) + } + }) + } +} + +func TestMultiEFACorrelatedCount(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "efa_rx_bytes") + require.NoError(t, err, "querying efa_rx_bytes") + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + require.True(t, len(multi) > 0, "No results from multi-EFA DRA node") + + var correlated int + for _, r := range multi { + if strings.HasPrefix(r.Labels.Resource["k8s.pod.name"], "efaburn") { + correlated++ + } + } + require.True(t, correlated >= 1, + "Expected at least 1 EFA correlated to efaburn, got %d", correlated) +} + +// TestMultiEFACorrelatedPodLabels validates that the correlated EFA result +// has the expected pod labels. +func TestMultiEFACorrelatedPodLabels(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "efa_rx_bytes") + require.NoError(t, err, "querying efa_rx_bytes") + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + + for _, r := range multi { + if strings.HasPrefix(r.Labels.Resource["k8s.pod.name"], "efaburn") { + require.Equal(t, "efaburn", r.Labels.Resource["k8s.container.name"], "efaburn container name") + require.Equal(t, "default", r.Labels.Resource["k8s.namespace.name"], "efaburn namespace") + return + } + } + t.Fatal("No efa_rx_bytes result correlated to efaburn pod") +} + +// expectedClaimedEFACount is how many of the node's EFA devices are claimed by a +// pod. efaburn (replicas: 1) requests 1 EFA via a ResourceClaimTemplate, so +// exactly one device is claimed and the remaining device(s) must stay unclaimed. +const expectedClaimedEFACount = 1 + +// TestMultiEFAClaimedVsUnclaimedCorrelation validates per-device pod correlation +// on a multi-EFA node whose devices are allocated via DRA: the device claimed by +// efaburn is correlated to that pod, and every remaining (unclaimed) device +// carries NO pod attributes. +// +// This is the DRA-path counterpart of the device-plugin regression guard. It +// exercises the processor's DRA correlation (ResourceClaim/ResourceSlice keying +// via dra.net/rdmaDevice) together with the groupbyattrs/efa split before the +// resource-level promote. Without correct per-device correlation, ALL of the +// node's EFA devices — including unclaimed ones — collapse onto a single pod; +// this test fails in that case because (a) more than expectedClaimedEFACount +// devices carry a pod, and (b) no device is left unclaimed. It also catches a +// single device attributed to multiple pods. +func TestMultiEFAClaimedVsUnclaimedCorrelation(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "efa_rx_bytes") + require.NoError(t, err, "querying efa_rx_bytes") + multi := filterByNodeLabel(results, multiEfaDraNodeLabel, "true") + require.NotEmpty(t, multi, + "No efa_rx_bytes results from multi-EFA DRA node (label %s)", multiEfaDraNodeLabel) + + // For each device, collect the distinct pods it is attributed to (empty = unclaimed). + devicePods := make(map[string]map[string]struct{}) + for _, r := range multi { + dev := getAnyValue(r, "aws.efa.device") + if dev == "" { + continue + } + if devicePods[dev] == nil { + devicePods[dev] = make(map[string]struct{}) + } + if pod := r.Labels.Resource["k8s.pod.name"]; pod != "" { + devicePods[dev][pod] = struct{}{} + } + } + require.Len(t, devicePods, expectedMultiEFACount, + "expected %d EFA devices on the node, got %d: %v", + expectedMultiEFACount, len(devicePods), deviceKeys(devicePods)) + + var claimed, unclaimed []string + for dev, pods := range devicePods { + switch len(pods) { + case 0: + unclaimed = append(unclaimed, dev) + case 1: + claimed = append(claimed, dev) + default: + // A single device correlated to multiple pods is itself a collapse symptom. + t.Errorf("EFA device %s correlated to multiple pods %v", dev, setKeys(pods)) + } + } + + // Claimed side: exactly the number of EFAs efaburn requested map to a pod. + require.Len(t, claimed, expectedClaimedEFACount, + "expected %d correlated (claimed) EFA device(s), got %d: %v "+ + "(collapse over-correlates unclaimed devices onto a pod)", + expectedClaimedEFACount, len(claimed), claimed) + + // Unclaimed side: the remaining devices must carry no pod — this is the + // coverage that distinguishes correct correlation from the collapse. + require.Len(t, unclaimed, expectedMultiEFACount-expectedClaimedEFACount, + "expected %d unclaimed EFA device(s) with no pod, got %d: %v", + expectedMultiEFACount-expectedClaimedEFACount, len(unclaimed), unclaimed) +} diff --git a/test/otel/multi_efa_dra/setup_test.go b/test/otel/multi_efa_dra/setup_test.go new file mode 100644 index 00000000..b522836d --- /dev/null +++ b/test/otel/multi_efa_dra/setup_test.go @@ -0,0 +1,98 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package multi_efa_dra + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +var ( + cfg otelmetrics.TestConfig + client *otelmetrics.OtelMetricsClient + queryCache *otelmetrics.QueryCache +) + +// Instance types in this cluster. +var clusterHostTypes = []string{"c6in.32xlarge"} + +func TestMain(m *testing.M) { + environment.RegisterEnvironmentMetaDataFlags() + flag.Parse() + env := environment.GetEnvironmentMetaData() + + region := env.Region + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + fmt.Fprintf(os.Stderr, "Region not set\n") + os.Exit(1) + } + + clusterName := env.EKSClusterName + if clusterName == "" { + clusterName = os.Getenv("CLUSTER_NAME") + } + if clusterName == "" { + fmt.Fprintf(os.Stderr, "Cluster name not set\n") + os.Exit(1) + } + + ctx := context.Background() + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + fmt.Fprintf(os.Stderr, "AWS config error: %v\n", err) + os.Exit(1) + } + stsClient := sts.NewFromConfig(awsCfg) + identity, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + fmt.Fprintf(os.Stderr, "STS GetCallerIdentity error: %v\n", err) + os.Exit(1) + } + + cfg = otelmetrics.TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + ClusterName: clusterName, + AccountID: *identity.Account, + SigningService: "monitoring", + } + + client, err = otelmetrics.NewClient(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Client error: %v\n", err) + os.Exit(1) + } + + hostMappings := []otelmetrics.SourceHostMapping{ + {Source: otelmetrics.SourceEFA, HostTypes: clusterHostTypes}, + } + + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, + otelmetrics.SourceMapping{Source: otelmetrics.SourceEFA, Metrics: efaMetrics}, + ) + + queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithHostTypes(clusterHostTypes), + otelmetrics.WithSourceRegistry(registry), + ) + + os.Exit(m.Run()) +} diff --git a/test/otel/neuron_dra/helpers_test.go b/test/otel/neuron_dra/helpers_test.go new file mode 100644 index 00000000..506781d0 --- /dev/null +++ b/test/otel/neuron_dra/helpers_test.go @@ -0,0 +1,87 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package neuron_dra + +import ( + "sort" + "strconv" + "strings" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +// escapePromQL escapes a string value for safe use inside PromQL label matches. +func escapePromQL(s string) string { + return strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) +} + +// filterByHostType returns results matching the given host.type. +func filterByHostType(results []otelmetrics.MetricResult, hostType string) []otelmetrics.MetricResult { + var out []otelmetrics.MetricResult + for _, r := range results { + r := r + if r.Labels.Resource["host.type"] == hostType { + out = append(out, r) + } + } + return out +} + +// uniqueDatapointValuesList returns the sorted unique non-empty values of a +// datapoint-level attribute across all results. +func uniqueDatapointValuesList(results []otelmetrics.MetricResult, key string) []string { + seen := make(map[string]struct{}) + for _, r := range results { + r := r + if v, ok := r.Labels.Datapoint[key]; ok && v != "" { + seen[v] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for v := range seen { + out = append(out, v) + } + sort.Strings(out) + return out +} + +// uniqueDatapointPairs collects distinct (a, b) pairs from two datapoint keys. +func uniqueDatapointPairs(results []otelmetrics.MetricResult, keyA, keyB string) [][2]string { + set := make(map[[2]string]struct{}) + for _, r := range results { + r := r + a := r.Labels.Datapoint[keyA] + b := r.Labels.Datapoint[keyB] + if a == "" || b == "" { + continue + } + set[[2]string{a, b}] = struct{}{} + } + out := make([][2]string, 0, len(set)) + for p := range set { + out = append(out, p) + } + return out +} + +// isIntLike returns true if s is a non-negative integer string. +func isIntLike(s string) bool { + if s == "" { + return false + } + _, err := strconv.Atoi(s) + return err == nil +} + +// setKeys returns the sorted keys of a string set (for messages). +func setKeys(m map[string]struct{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/test/otel/neuron_dra/metrics_test.go b/test/otel/neuron_dra/metrics_test.go new file mode 100644 index 00000000..d20a38e0 --- /dev/null +++ b/test/otel/neuron_dra/metrics_test.go @@ -0,0 +1,18 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package neuron_dra + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +// Neuron metric definitions (same source names as the device-plugin neuron package; +// OTel preserves raw neuron-monitor names regardless of allocation mechanism). +// neuroncore_utilization_ratio is the per-core metric the correlation tests use. +var neuronMetrics = []otelmetrics.MetricDefinition{ + {Name: "neuroncore_utilization_ratio", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "1"}, + {Name: "neuroncore_memory_usage_model_shared_scratchpad", MetricType: "gauge", Scope: otelmetrics.ScopePod, Unit: "By"}, + {Name: "neuron_runtime_memory_used_bytes", MetricType: "gauge", Scope: otelmetrics.ScopePod, ExpectedLabels: []string{"memory_location"}, Unit: "By"}, + {Name: "execution_latency_seconds", MetricType: "gauge", Scope: otelmetrics.ScopePod, ExpectedLabels: []string{"percentile"}, Unit: "s"}, +} diff --git a/test/otel/neuron_dra/neuron_dra_test.go b/test/otel/neuron_dra/neuron_dra_test.go new file mode 100644 index 00000000..45d0f808 --- /dev/null +++ b/test/otel/neuron_dra/neuron_dra_test.go @@ -0,0 +1,219 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +// Package neuron_dra validates per-device/per-core Neuron pod correlation on the +// Dynamic Resource Allocation (DRA) path — the counterpart of the device-plugin +// `neuron` package's multi_device tests. Neuron devices are allocated via the +// AWS Neuron DRA driver (DeviceClass neuron.aws.com, driver neuron.aws.com) through +// a ResourceClaimTemplate instead of the aws.amazon.com/neuron device-plugin +// resource. The emitted metrics are identical, so the assertions mirror the +// device-plugin tests. +// +// Cluster topology: +// - 1x trn1.2xlarge = 1 Neuron (Trainium) device × 2 cores = 2 cores +// - Neuron exposed via the Neuron DRA driver, not the device plugin +// - A burn workload (neuron-burn-dra) claims 1 whole device via DRA +// +// Why Trainium (trn1) and not Inferentia (inf2): the AWS Neuron DRA driver +// (neuron-helm-chart, driver image 1.2.0) supports Trainium only — it explicitly +// rejects inf1/inf2 at device discovery ("unsupported instance type"). trn1.2xlarge +// is the smallest, most reliably-available Trainium instance (1 device / 2 cores), +// which is enough to exercise the DRA correlation code path end to end: a claimed +// device's two cores must both attribute to the claiming pod, and to no other pod. +// The multi-device "1 claimed + N-1 unclaimed" matrix is covered by the +// device-plugin multi_device_test; here the device count is 1 so the whole node's +// single device is the claimed device. +package neuron_dra + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + neuronInstanceType = "trn1.2xlarge" + expectedNeuronDeviceCount = 1 + expectedNeuronCoreCount = 2 + expectedCoresPerDevice = 2 + + // burnPodPrefix is the DRA burn workload that claims 1 whole Neuron device + // (= 2 cores) via a ResourceClaimTemplate. + burnPodPrefix = "neuron-burn-dra" + // A claim for 1 whole device yields 2 cores on 1 device. + expectedClaimedDevices = 1 + expectedClaimedCores = 2 +) + +// TestNeuronDRADeviceCount validates the node exposes the expected Neuron devices. +func TestNeuronDRADeviceCount(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "neuroncore_utilization_ratio") + require.NoError(t, err, "querying neuroncore_utilization_ratio") + require.NotEmpty(t, results, "neuroncore_utilization_ratio not available") + + node := filterByHostType(results, neuronInstanceType) + require.True(t, len(node) > 0, "No results from %s node", neuronInstanceType) + + devices := uniqueDatapointValuesList(node, "aws.neuron.device") + require.Equal(t, expectedNeuronDeviceCount, len(devices), + "Expected %d distinct aws.neuron.device on %s, got %d: %v", + expectedNeuronDeviceCount, neuronInstanceType, len(devices), devices) +} + +// TestNeuronDRACoreCount validates the total (device, core) pair count. +func TestNeuronDRACoreCount(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "neuroncore_utilization_ratio") + require.NoError(t, err, "querying neuroncore_utilization_ratio") + require.NotEmpty(t, results, "neuroncore_utilization_ratio not available") + + node := filterByHostType(results, neuronInstanceType) + require.True(t, len(node) > 0, "No results from %s node", neuronInstanceType) + + pairs := uniqueDatapointPairs(node, "aws.neuron.device", "aws.neuron.core") + require.Equal(t, expectedNeuronCoreCount, len(pairs), + "Expected %d (device, core) pairs on %s, got %d", + expectedNeuronCoreCount, neuronInstanceType, len(pairs)) +} + +// TestNeuronDRACoresPerDevice validates each device exposes the expected cores. +func TestNeuronDRACoresPerDevice(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "neuroncore_utilization_ratio") + require.NoError(t, err, "querying neuroncore_utilization_ratio") + require.NotEmpty(t, results, "neuroncore_utilization_ratio not available") + + node := filterByHostType(results, neuronInstanceType) + require.True(t, len(node) > 0, "No results from %s node", neuronInstanceType) + + coresByDevice := make(map[string]map[string]struct{}) + for _, r := range node { + r := r + dev := r.Labels.Datapoint["aws.neuron.device"] + core := r.Labels.Datapoint["aws.neuron.core"] + if dev == "" || core == "" { + continue + } + if coresByDevice[dev] == nil { + coresByDevice[dev] = make(map[string]struct{}) + } + coresByDevice[dev][core] = struct{}{} + } + require.True(t, len(coresByDevice) > 0, "No devices with core metrics on %s", neuronInstanceType) + for dev, cores := range coresByDevice { + require.Equal(t, expectedCoresPerDevice, len(cores), + "Neuron device %s: expected %d cores, got %d", dev, expectedCoresPerDevice, len(cores)) + } +} + +// TestNeuronDRADeviceIndicesAreIntegers validates device and core are integers. +func TestNeuronDRADeviceIndicesAreIntegers(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "neuroncore_utilization_ratio") + require.NoError(t, err, "querying neuroncore_utilization_ratio") + node := filterByHostType(results, neuronInstanceType) + require.True(t, len(node) > 0, "No results from %s node", neuronInstanceType) + + for _, r := range node { + r := r + dev := r.Labels.Datapoint["aws.neuron.device"] + core := r.Labels.Datapoint["aws.neuron.core"] + require.True(t, isIntLike(dev), "aws.neuron.device should be integer, got '%s'", dev) + require.True(t, isIntLike(core), "aws.neuron.core should be integer, got '%s'", core) + } +} + +// TestNeuronDRAClaimedDeviceCorrelation is the DRA-path per-device correlation guard. +// The burn pod claims exactly 1 whole Neuron device via DRA, so both of that device's +// cores must attribute to that single pod — and to exactly one pod. This fails if DRA +// correlation collapses cores onto the wrong pod, attributes an unclaimed core to a +// pod, or splits a device across pods. On this single-device node the node's one +// device is the claimed device. +func TestNeuronDRAClaimedDeviceCorrelation(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), "neuroncore_utilization_ratio") + require.NoError(t, err, "querying neuroncore_utilization_ratio") + node := filterByHostType(results, neuronInstanceType) + require.NotEmpty(t, node, "No neuroncore_utilization_ratio results from %s node", neuronInstanceType) + + // For each device, collect the distinct pods its cores are attributed to + // (empty = unclaimed), and count how many of its cores carry the burn pod. + devicePods := make(map[string]map[string]struct{}) + claimedCoresByDevice := make(map[string]map[string]struct{}) + for _, r := range node { + r := r + dev := r.Labels.Datapoint["aws.neuron.device"] + core := r.Labels.Datapoint["aws.neuron.core"] + if dev == "" { + continue + } + if devicePods[dev] == nil { + devicePods[dev] = make(map[string]struct{}) + } + pod := r.Labels.Resource["k8s.pod.name"] + if pod == "" { + continue + } + devicePods[dev][pod] = struct{}{} + if len(pod) >= len(burnPodPrefix) && pod[:len(burnPodPrefix)] == burnPodPrefix { + if claimedCoresByDevice[dev] == nil { + claimedCoresByDevice[dev] = make(map[string]struct{}) + } + claimedCoresByDevice[dev][core] = struct{}{} + } + } + require.Len(t, devicePods, expectedNeuronDeviceCount, + "expected %d Neuron device(s) on the node, got %d", expectedNeuronDeviceCount, len(devicePods)) + + var claimedDevices []string + for dev, pods := range devicePods { + switch len(pods) { + case 0: + // No pod on any core of this device. + case 1: + claimedDevices = append(claimedDevices, dev) + default: + // A single device whose cores span multiple pods is a collapse symptom. + t.Errorf("Neuron device %s correlated to multiple pods %v", dev, setKeys(pods)) + } + } + + // Exactly 1 device correlated, and exactly 2 of its cores → the burn pod. + require.Len(t, claimedDevices, expectedClaimedDevices, + "expected %d claimed Neuron device(s), got %d: %v", + expectedClaimedDevices, len(claimedDevices), claimedDevices) + for _, dev := range claimedDevices { + require.Len(t, claimedCoresByDevice[dev], expectedClaimedCores, + fmt.Sprintf("expected %d cores of device %s correlated to %s*, got %d", + expectedClaimedCores, dev, burnPodPrefix, len(claimedCoresByDevice[dev]))) + } +} + +// TestNeuronDRABurnPodLabels validates the correlated burn pod's labels via a +// targeted PromQL query (avoids stale series in the shared OTLP store). +func TestNeuronDRABurnPodLabels(t *testing.T) { + t.Parallel() + ctx := context.Background() + promql := fmt.Sprintf( + `neuroncore_utilization_ratio{"@resource.k8s.cluster.name"="%s","@resource.k8s.pod.name"=~"%s.*"}`, + escapePromQL(cfg.ClusterName), burnPodPrefix) + burn, err := client.Query(ctx, promql) + require.NoError(t, err, "querying neuroncore_utilization_ratio for %s", burnPodPrefix) + require.Equal(t, expectedClaimedCores, len(burn), + "Expected %d cores correlated to %s (1 whole device × 2 cores), got %d", + expectedClaimedCores, burnPodPrefix, len(burn)) + + devices := make(map[string]struct{}) + for _, r := range burn { + r := r + devices[r.Labels.Datapoint["aws.neuron.device"]] = struct{}{} + require.Equal(t, "default", r.Labels.Resource["k8s.namespace.name"], "%s namespace", burnPodPrefix) + } + require.Len(t, devices, expectedClaimedDevices, + "Expected %s cores on %d device, got %d", burnPodPrefix, expectedClaimedDevices, len(devices)) +} diff --git a/test/otel/neuron_dra/setup_test.go b/test/otel/neuron_dra/setup_test.go new file mode 100644 index 00000000..ac18d598 --- /dev/null +++ b/test/otel/neuron_dra/setup_test.go @@ -0,0 +1,98 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package neuron_dra + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +var ( + cfg otelmetrics.TestConfig + client *otelmetrics.OtelMetricsClient + queryCache *otelmetrics.QueryCache +) + +// The DRA cluster has a single Neuron (Trainium) node (trn1.2xlarge). +var clusterHostTypes = []string{neuronInstanceType} + +func TestMain(m *testing.M) { + environment.RegisterEnvironmentMetaDataFlags() + flag.Parse() + env := environment.GetEnvironmentMetaData() + + region := env.Region + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + fmt.Fprintf(os.Stderr, "Region not set\n") + os.Exit(1) + } + + clusterName := env.EKSClusterName + if clusterName == "" { + clusterName = os.Getenv("CLUSTER_NAME") + } + if clusterName == "" { + fmt.Fprintf(os.Stderr, "Cluster name not set\n") + os.Exit(1) + } + + ctx := context.Background() + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + fmt.Fprintf(os.Stderr, "AWS config error: %v\n", err) + os.Exit(1) + } + stsClient := sts.NewFromConfig(awsCfg) + identity, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + fmt.Fprintf(os.Stderr, "STS GetCallerIdentity error: %v\n", err) + os.Exit(1) + } + + cfg = otelmetrics.TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + ClusterName: clusterName, + AccountID: *identity.Account, + SigningService: "monitoring", + } + + client, err = otelmetrics.NewClient(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Client error: %v\n", err) + os.Exit(1) + } + + hostMappings := []otelmetrics.SourceHostMapping{ + {Source: otelmetrics.SourceNeuron, HostTypes: clusterHostTypes}, + } + + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, + otelmetrics.SourceMapping{Source: otelmetrics.SourceNeuron, Metrics: neuronMetrics}, + ) + + queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithHostTypes(clusterHostTypes), + otelmetrics.WithSourceRegistry(registry), + ) + + os.Exit(m.Run()) +}