From c340d33707f473f72ec7ce72f68ad983a7b8c3ab Mon Sep 17 00:00:00 2001 From: Spandan Naik Date: Wed, 7 Oct 2026 13:22:20 +0000 Subject: [PATCH] test(neuron): integration coverage + fixture for multi-runtime per-core metrics On a node with more than one Neuron runtime, neuron-monitor reports every core from every runtime, separated only by the runtime tag. These tests verify that per-core metrics keep one series per (core, runtime tag), so a busy core is never reported as idle. Coverage for https://github.com/aws/amazon-cloudwatch-agent/pull/2263 and https://github.com/aws-observability/helm-charts/pull/372. Fixture (terraform/eks/daemon/otel-neuron/main.tf): - neuron-burn-peer, a second burn Deployment taking the other core of the inf2.xlarge, co-located with neuron-burn-core through podAffinity. It is not named neuron-burn-core-* and has no neuron-test label, so the existing burn-core and idle-node tests are unaffected. - NEURON_PROCESS_TAG set to the pod name on both burn pods. The runtime tag defaults to the process PID, which is 1 in every container. - wait_neuron_monitor waits for both burn pods to be Ready and fails unless they share a node; the validator also waits for neuron-burn-peer's burn loop. Tests (test/otel/neuron/multi_runtime_test.go): 8 tests covering busy-core values per runtime tag, the cores x runtime-tag series count, unique core identity, one owning runtime per core, core-to-pod attribution, the runtime tag as a non-empty resource attribute, and the presence of both burn workloads. --- terraform/eks/daemon/otel-neuron/main.tf | 169 ++++++++- test/otel/neuron/multi_runtime_test.go | 455 +++++++++++++++++++++++ test/otel/neuron/neuron_test.go | 2 +- 3 files changed, 618 insertions(+), 8 deletions(-) create mode 100644 test/otel/neuron/multi_runtime_test.go diff --git a/terraform/eks/daemon/otel-neuron/main.tf b/terraform/eks/daemon/otel-neuron/main.tf index d292eceb9..9fcee0a4f 100644 --- a/terraform/eks/daemon/otel-neuron/main.tf +++ b/terraform/eks/daemon/otel-neuron/main.tf @@ -336,6 +336,115 @@ resource "null_resource" "neuron_burn_core" { elapsed = time.time() - start iteration += 1 print(f"Iteration {iteration}: 1000 inferences in {elapsed:.2f}s") + env: + # The runtime tag defaults to the PID, which is 1 in every container. + - name: NEURON_PROCESS_TAG + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: + limits: + aws.amazon.com/neuroncore: "1" + requests: + cpu: "1" + memory: 4Gi + EOF + EOT + } +} + +# --- neuron-burn-peer Deployment: the SECOND runtime on the workload node --- +# +# Gives the node two Neuron runtimes burning different cores, the shape the +# multi-runtime tests need (test/otel/neuron/multi_runtime_test.go). +# inf2.xlarge has 1 device x 2 cores; burn-core takes one, this takes the other. +# +# Three constraints, each of which breaks an existing test if changed: +# - NOT named neuron-burn-core-*: TestNeuronBurnWorkloadLabels and +# TestNeuronBurnCorePodColor match HasPrefix(pod, "neuron-burn-core"). +# - no `neuron-test: "true"` label: burn-core's podAntiAffinity targets it, which +# would make the two mutually exclusive on a host and push burn-core onto the +# idle node, breaking the idle-node tests. +# - podAffinity, not just nodeSelector: both node groups use var.instance_type, so +# a nodeSelector alone could land this on the idle node. +# +# Sits Pending on first apply until burn-core is scheduled; the scheduler retries. + +resource "null_resource" "neuron_burn_peer" { + depends_on = [ + helm_release.neuron_device_plugin, + null_resource.kubectl, + null_resource.neuron_burn_core, + ] + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF' | kubectl apply -f - + apiVersion: apps/v1 + kind: Deployment + metadata: + name: neuron-burn-peer + namespace: default + spec: + replicas: 1 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 300 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + selector: + matchLabels: + app: neuron-burn-peer + template: + metadata: + labels: + app: neuron-burn-peer + ci-test.example.com/pod-color: green + spec: + tolerations: + - key: aws.amazon.com/neuron + operator: Exists + effect: NoSchedule + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: ["neuron-burn-core"] + topologyKey: kubernetes.io/hostname + nodeSelector: + node.kubernetes.io/instance-type: ${var.instance_type} + containers: + - name: neuron-burn + image: public.ecr.aws/neuron/pytorch-inference-neuronx:2.1.2-neuronx-py310-sdk2.20.2-ubuntu20.04 + command: ["python3", "-c"] + args: + - | + import torch + import torch_neuronx + import time + print("Compiling neuron trace (this takes a minute)...") + x = torch.randn(256, 256) + model = torch.nn.Linear(256, 256, bias=False) + traced = torch_neuronx.trace(model, x) + print("Trace compiled. Starting burn loop...") + iteration = 0 + while True: + start = time.time() + for _ in range(1000): + _ = traced(x) + elapsed = time.time() - start + iteration += 1 + print(f"Iteration {iteration}: 1000 inferences in {elapsed:.2f}s") + env: + # The runtime tag defaults to the PID, which is 1 in every container. + - name: NEURON_PROCESS_TAG + valueFrom: + fieldRef: + fieldPath: metadata.name resources: limits: aws.amazon.com/neuroncore: "1" @@ -501,10 +610,41 @@ resource "null_resource" "restart_pods" { # --- Wait for neuron-monitor pods to be ready --- resource "null_resource" "wait_neuron_monitor" { - depends_on = [null_resource.restart_pods, null_resource.neuron_burn_core, null_resource.neuron_burn_multi_device] - triggers = { timestamp = timestamp() } + depends_on = [ + null_resource.restart_pods, + null_resource.neuron_burn_core, + null_resource.neuron_burn_peer, + null_resource.neuron_burn_multi_device, + ] + triggers = { timestamp = timestamp() } provisioner "local-exec" { command = <<-EOT + # A runtime_tag only exists once the runtime is up, and the trace compile takes + # ~a minute, so without this the tests race the fixture and see one runtime. + # Waits on the pods, not `rollout status`: a cold pull of the PyTorch-Neuron image + # can outlast the deployments' 300s progressDeadlineSeconds. + echo "Waiting for burn pods to become Ready..." + for d in neuron-burn-core neuron-burn-peer; do + kubectl -n default wait --for=condition=Ready pod -l app=$d --timeout=900s || { + echo "ERROR: pods for deployment/$d did not become Ready" + kubectl -n default describe pod -l app=$d | tail -40 || true + kubectl -n default get pods -l app=$d -o wide || true + exit 1 + } + done + + # A silent split across nodes would make every multi-runtime test vacuous. + NODES=$(kubectl -n default get pods -l 'app in (neuron-burn-core,neuron-burn-peer)' \ + -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | grep -c . || true) + if [ "$NODES" != "1" ]; then + echo "ERROR: neuron-burn-core and neuron-burn-peer are on $NODES nodes, expected 1." + echo "The multi-runtime tests cannot run unless both" + echo "runtimes share a node. Check the podAffinity on neuron-burn-peer." + kubectl -n default get pods -l 'app in (neuron-burn-core,neuron-burn-peer)' -o wide || true + exit 1 + fi + echo "Both burn runtimes are co-located on one node." + echo "Waiting for neuron-monitor pods to be ready..." READY=0 for i in $(seq 1 30); do @@ -538,6 +678,7 @@ resource "null_resource" "validator" { echo "This can take up to 15 minutes on a cold start due to PyTorch-Neuron image pull + trace compilation." echo "Readiness signal: 'Iteration N' log lines from the active burn loop." CORE_READY=0 + PEER_READY=0 MULTI_READY=0 # 90 iterations × 10s = 15 minutes max. for i in $(seq 1 90); do @@ -545,28 +686,41 @@ resource "null_resource" "validator" { # the trace is compiled. This is a more robust readiness signal than # grepping for "Trace compiled" which scrolls off the tail quickly. CORE_READY=$(kubectl logs -n default -l app=neuron-burn-core --tail=5 2>/dev/null | grep -c "^Iteration " || true) + # neuron-burn-peer is the second runtime on the core node. wait_neuron_monitor + # only gets it to Available (container Running); the trace compile that starts + # it emitting runtime_tag takes ~1 more minute, so gate on it here too -- + # otherwise the multi-runtime tests can run against a single-runtime surface. + PEER_READY=$(kubectl logs -n default -l app=neuron-burn-peer --tail=5 2>/dev/null | grep -c "^Iteration " || true) MULTI_READY=$(kubectl logs -n default -l app=neuron-burn-multi --tail=5 2>/dev/null | grep -c "^Iteration " || true) - if [ "$CORE_READY" -gt 0 ] && [ "$MULTI_READY" -gt 0 ]; then - echo "Neuron runtime active on both burn workloads (after $((i*10))s)" + if [ "$CORE_READY" -gt 0 ] && [ "$PEER_READY" -gt 0 ] && [ "$MULTI_READY" -gt 0 ]; then + echo "Neuron runtime active on all burn workloads (after $((i*10))s)" break fi # Every 60s dump pod status so we can see image-pull vs crash vs runtime-init. if [ $((i % 6)) -eq 0 ]; then - echo "--- Attempt $i ($((i*10))s elapsed): core_iter_lines=$CORE_READY multi_iter_lines=$MULTI_READY ---" + echo "--- Attempt $i ($((i*10))s elapsed): core_iter_lines=$CORE_READY peer_iter_lines=$PEER_READY multi_iter_lines=$MULTI_READY ---" kubectl get pods -n default -l neuron-test=true -o wide 2>&1 | head -10 || true + # neuron-burn-peer carries no neuron-test label (the anti-affinity on + # neuron-burn-core targets it), so the dump above omits it. + kubectl get pods -n default -l app=neuron-burn-peer -o wide 2>&1 | head -5 || true fi sleep 10 done - if [ "$CORE_READY" -eq 0 ] || [ "$MULTI_READY" -eq 0 ]; then - echo "ERROR: Neuron burn loop not active after 15 minutes (core_iter=$CORE_READY multi_iter=$MULTI_READY)" + if [ "$CORE_READY" -eq 0 ] || [ "$PEER_READY" -eq 0 ] || [ "$MULTI_READY" -eq 0 ]; then + echo "ERROR: Neuron burn loop not active after 15 minutes (core_iter=$CORE_READY peer_iter=$PEER_READY multi_iter=$MULTI_READY)" echo "=== Final pod status ===" kubectl get pods -n default -l neuron-test=true -o wide 2>&1 || true + kubectl get pods -n default -l app=neuron-burn-peer -o wide 2>&1 || true echo "=== burn-core events ===" kubectl describe pod -n default -l app=neuron-burn-core 2>&1 | tail -40 || true + echo "=== burn-peer events ===" + kubectl describe pod -n default -l app=neuron-burn-peer 2>&1 | tail -40 || true echo "=== burn-multi events ===" kubectl describe pod -n default -l app=neuron-burn-multi 2>&1 | tail -40 || true echo "=== burn-core logs ===" kubectl logs -n default -l app=neuron-burn-core --tail=40 2>&1 || true + echo "=== burn-peer logs ===" + kubectl logs -n default -l app=neuron-burn-peer --tail=40 2>&1 || true echo "=== burn-multi logs ===" kubectl logs -n default -l app=neuron-burn-multi --tail=40 2>&1 || true exit 1 @@ -574,6 +728,7 @@ resource "null_resource" "validator" { echo "=== Burn workload pod status ===" kubectl get pods -n default -l neuron-test=true -o wide + kubectl get pods -n default -l app=neuron-burn-peer -o wide echo "Waiting 6 minutes for metrics to propagate (covers Zeus 5-min staleness window)..." sleep 360 diff --git a/test/otel/neuron/multi_runtime_test.go b/test/otel/neuron/multi_runtime_test.go new file mode 100644 index 000000000..a2336bf7e --- /dev/null +++ b/test/otel/neuron/multi_runtime_test.go @@ -0,0 +1,455 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +// Multi-runtime Neuron tests. +// +// On a node with more than one Neuron runtime, neuron-monitor reports every core +// from every runtime: the runtime that owns a core reports its real value and the +// others report 0 for it. The runtime tag (@resource.aws.neuron.runtime.tag) is the +// only attribute that separates those readings. These tests verify that per-core +// metrics keep one series per (core, runtime tag), so a busy core is never +// reported as idle and per-runtime attribution is preserved. +// +// Fixture: terraform/eks/daemon/otel-neuron/main.tf co-locates neuron-burn-core +// and neuron-burn-peer on one inf2.xlarge (1 device x 2 cores), each holding +// aws.amazon.com/neuroncore: "1". +// +// Label semantics, easy to get backwards: k8s.pod.name is the pod that OWNS THE +// CORE (awsdevicepodcorrelation maps core -> pod); runtime.tag is the runtime that +// PRODUCED THE READING. They cross — runtime B emits a zero for core 0, correlated +// to pod A. So core -> pod is 1:1; tag -> pod is not. + +package neuron + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +const runtimeTagResourceKey = "aws.neuron.runtime.tag" + +// Runtime tag value that indicates the runtime dimension was aggregated away. +const runtimeTagFlattenedSentinel = "DEFAULT" + +const ( + burnCorePodPrefix = "neuron-burn-core" + burnPeerPodPrefix = "neuron-burn-peer" +) + +const fixtureRequirement = "the otel-neuron terraform fixture must have " + + "neuron-burn-core AND neuron-burn-peer Running and co-located on one " + + "inf2.xlarge node, each holding aws.amazon.com/neuroncore: \"1\" and with a " + + "distinct NEURON_PROCESS_TAG" + +type coreIdentity struct { + node string + device string + core string +} + +func (c coreIdentity) String() string { + return fmt.Sprintf("%s device=%s core=%s", c.node, c.device, c.core) +} + +// ok is false for runtime-level metrics, which have no core dimension. +func coreIdentityOf(r otelmetrics.MetricResult) (coreIdentity, bool) { + node := r.Labels.Resource["k8s.node.name"] + device, hasDevice := r.Labels.Datapoint["aws.neuron.device"] + core, hasCore := r.Labels.Datapoint["aws.neuron.core"] + if node == "" || !hasDevice || !hasCore { + return coreIdentity{}, false + } + return coreIdentity{node: node, device: device, core: core}, true +} + +func runtimeTagsByNode(results []otelmetrics.MetricResult) map[string]map[string]struct{} { + byNode := make(map[string]map[string]struct{}) + for _, r := range results { + r := r + node := r.Labels.Resource["k8s.node.name"] + tag := r.Labels.Resource[runtimeTagResourceKey] + if node == "" || tag == "" { + continue + } + if byNode[node] == nil { + byNode[node] = make(map[string]struct{}) + } + byNode[node][tag] = struct{}{} + } + return byNode +} + +func multiRuntimeNodes(results []otelmetrics.MetricResult) []string { + var nodes []string + for node, tags := range runtimeTagsByNode(results) { + if len(tags) >= 2 { + nodes = append(nodes, node) + } + } + sort.Strings(nodes) + return nodes +} + +func resultsForNode(results []otelmetrics.MetricResult, node string) []otelmetrics.MetricResult { + var out []otelmetrics.MetricResult + for _, r := range results { + r := r + if r.Labels.Resource["k8s.node.name"] == node { + out = append(out, r) + } + } + return out +} + +func sortedKeys(set map[string]struct{}) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedTags(results []otelmetrics.MetricResult) []string { + set := make(map[string]struct{}) + for _, r := range results { + r := r + if tag := r.Labels.Resource[runtimeTagResourceKey]; tag != "" { + set[tag] = struct{}{} + } + } + return sortedKeys(set) +} + +func tagSummary(byNode map[string]map[string]struct{}) map[string][]string { + out := make(map[string][]string, len(byNode)) + for node, tags := range byNode { + out[node] = sortedKeys(tags) + } + return out +} + +// multiRuntimeResults returns the results for metricName and the nodes that report +// at least two runtime tags. It fails rather than skips when there are none: the +// fixture is part of the cluster definition, so a skip would silently disable +// every test in this file. A collapsed runtime dimension and a single-runtime +// fixture look the same here (one tag per node), so the message names both. +func multiRuntimeResults(t *testing.T, metricName string) ([]otelmetrics.MetricResult, []string) { + t.Helper() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (no Neuron nodes?)", metricName) + + nodes := multiRuntimeNodes(results) + require.NotEmpty(t, nodes, + "%s: no node reports >= 2 distinct @resource.%s. Observed tags per node: %v.\n"+ + "TWO causes look identical here, check both:\n"+ + " 1. RUNTIME DIMENSION COLLAPSED — the agent merged runtimes, so one tag "+ + "overwrote the others and half the per-core values were replaced by zero. "+ + "Confirm with `kubectl -n amazon-cloudwatch get cm cloudwatch-agent -o yaml`: "+ + "groupbyattrs/cw_k8s_ci_v0_neuron must include the runtime_tag key, and "+ + "transform/cw_k8s_ci_v0_neuron_promote must run in `context: resource`.\n"+ + " 2. THE FIXTURE — only one Neuron runtime is running, so there is nothing "+ + "to collapse. %s", + metricName, runtimeTagResourceKey, tagSummary(runtimeTagsByNode(results)), fixtureRequirement) + return results, nodes +} + +// TestMultiRuntimeDistinctWorkloadsOwnDistinctCores verifies values: two workloads +// burning two cores yield two non-zero core readings, attributed to two runtime tags. +func TestMultiRuntimeDistinctWorkloadsOwnDistinctCores(t *testing.T) { + t.Parallel() + const metricName = "neuroncore_utilization_ratio" + results, nodes := multiRuntimeResults(t, metricName) + + for _, node := range nodes { + node := node + t.Run(node, func(t *testing.T) { + t.Parallel() + busyCores := make(map[coreIdentity]struct{}) + busyTags := make(map[string]struct{}) + var detail []string + + for _, r := range resultsForNode(results, node) { + r := r + if r.Value <= 0 { + continue + } + id, ok := coreIdentityOf(r) + if !ok { + continue + } + busyCores[id] = struct{}{} + if tag := r.Labels.Resource[runtimeTagResourceKey]; tag != "" { + busyTags[tag] = struct{}{} + } + detail = append(detail, fmt.Sprintf("core=%s tag=%s pod=%s value=%.4f", + id.core, r.Labels.Resource[runtimeTagResourceKey], + r.Labels.Resource["k8s.pod.name"], r.Value)) + } + sort.Strings(detail) + + require.GreaterOrEqual(t, len(busyCores), 2, + "%s on %s: expected >= 2 cores with non-zero utilization, got %d. Exactly "+ + "one busy core means another runtime's 0 is shadowing a real reading. "+ + "Non-zero series: %v. %s", + metricName, node, len(busyCores), detail, fixtureRequirement) + + // No assertion on distinct pod count: one pod can hold both cores, so two + // runtimes need not be two pods. + require.GreaterOrEqual(t, len(busyTags), 2, + "%s on %s: %d busy cores attributed to only %d runtime tag(s) %v — the "+ + "runtime dimension collapsed. Non-zero series: %v", + metricName, node, len(busyCores), len(busyTags), sortedKeys(busyTags), detail) + }) + } +} + +// TestMultiRuntimeFixtureIsBothBurnWorkloads verifies that both burn workloads are +// present on the multi-runtime node, so the other tests in this file exercise two +// runtimes. +func TestMultiRuntimeFixtureIsBothBurnWorkloads(t *testing.T) { + t.Parallel() + const metricName = "neuroncore_utilization_ratio" + results, nodes := multiRuntimeResults(t, metricName) + + for _, node := range nodes { + node := node + t.Run(node, func(t *testing.T) { + t.Parallel() + pods := make(map[string]struct{}) + for _, r := range resultsForNode(results, node) { + r := r + if pod := r.Labels.Resource["k8s.pod.name"]; pod != "" { + pods[pod] = struct{}{} + } + } + names := sortedKeys(pods) + + var sawCore, sawPeer bool + for _, p := range names { + if strings.HasPrefix(p, burnPeerPodPrefix) { + sawPeer = true + } else if strings.HasPrefix(p, burnCorePodPrefix) { + sawCore = true + } + } + require.True(t, sawCore, + "no pod prefixed %q among %v on multi-runtime node %s", burnCorePodPrefix, names, node) + require.True(t, sawPeer, + "no pod prefixed %q among %v on multi-runtime node %s. %s", + burnPeerPodPrefix, names, node, fixtureRequirement) + }) + } +} + +// TestMultiRuntimeCoreCrossProductPreserved verifies structure: every runtime +// reports every core, so the series count equals cores x runtime tags. This holds +// regardless of load, including on an idle node. +func TestMultiRuntimeCoreCrossProductPreserved(t *testing.T) { + t.Parallel() + for _, md := range neuronCoreLevelMetrics { + md := md + t.Run(md.Name, func(t *testing.T) { + t.Parallel() + results, nodes := multiRuntimeResults(t, md.Name) + + for _, node := range nodes { + node := node + nodeResults := resultsForNode(results, node) + tags := sortedTags(nodeResults) + + cores := make(map[coreIdentity]struct{}) + pairs := make(map[string]struct{}) + for _, r := range nodeResults { + r := r + id, ok := coreIdentityOf(r) + if !ok { + continue + } + cores[id] = struct{}{} + pairs[fmt.Sprintf("%s|%s", id, r.Labels.Resource[runtimeTagResourceKey])] = struct{}{} + } + + require.NotEmpty(t, cores, "%s: no core-level results on node %s", md.Name, node) + want := len(cores) * len(tags) + require.Equal(t, want, len(pairs), + "%s on %s: expected %d (core x runtime-tag) series for %d cores x %d "+ + "runtimes %v, got %d. Fewer means one runtime's datapoints are "+ + "shadowing another's.", + md.Name, node, want, len(cores), len(tags), tags, len(pairs)) + } + }) + } +} + +// TestMultiRuntimeCoreIdentityIsUnique verifies that (node, device, core, runtime +// tag) identifies exactly one series. It is narrower than +// TestNeuronNoDuplicateSeries, so an unrelated label cannot make two series look +// distinct. +func TestMultiRuntimeCoreIdentityIsUnique(t *testing.T) { + t.Parallel() + for _, md := range neuronCoreLevelMetrics { + md := md + t.Run(md.Name, func(t *testing.T) { + t.Parallel() + results, nodes := multiRuntimeResults(t, md.Name) + + for _, node := range nodes { + node := node + seen := make(map[string]int) + for _, r := range resultsForNode(results, node) { + r := r + id, ok := coreIdentityOf(r) + if !ok { + continue + } + seen[fmt.Sprintf("%s|tag=%s", id, r.Labels.Resource[runtimeTagResourceKey])]++ + } + for key, count := range seen { + require.Equal(t, 1, count, + "%s on %s: %d series share the identity %s", md.Name, node, count, key) + } + } + }) + } +} + +// TestMultiRuntimeOneRuntimeOwnsEachCore verifies that at most one runtime reports a +// non-zero value for each core, since one runtime holds a core. +func TestMultiRuntimeOneRuntimeOwnsEachCore(t *testing.T) { + t.Parallel() + const metricName = "neuroncore_utilization_ratio" + results, nodes := multiRuntimeResults(t, metricName) + + for _, node := range nodes { + node := node + t.Run(node, func(t *testing.T) { + t.Parallel() + nonZeroPerCore := make(map[coreIdentity][]string) + for _, r := range resultsForNode(results, node) { + r := r + if r.Value <= 0 { + continue + } + id, ok := coreIdentityOf(r) + if !ok { + continue + } + nonZeroPerCore[id] = append(nonZeroPerCore[id], + r.Labels.Resource[runtimeTagResourceKey]) + } + for id, tags := range nonZeroPerCore { + sort.Strings(tags) + require.LessOrEqual(t, len(tags), 1, + "%s on %s: %d runtime tags %v report non-zero for %s", + metricName, node, len(tags), tags, id) + } + }) + } +} + +// TestMultiRuntimeCoreToPodIsOneToOne verifies that each core is attributed to +// exactly one pod. Runtime tag -> pod is not 1:1; see the label semantics above. +func TestMultiRuntimeCoreToPodIsOneToOne(t *testing.T) { + t.Parallel() + for _, md := range neuronCoreLevelMetrics { + md := md + t.Run(md.Name, func(t *testing.T) { + t.Parallel() + results, nodes := multiRuntimeResults(t, md.Name) + + for _, node := range nodes { + node := node + coreToPods := make(map[coreIdentity]map[string]struct{}) + for _, r := range resultsForNode(results, node) { + r := r + id, ok := coreIdentityOf(r) + if !ok { + continue + } + pod := r.Labels.Resource["k8s.pod.name"] + if pod == "" { + continue + } + if coreToPods[id] == nil { + coreToPods[id] = make(map[string]struct{}) + } + coreToPods[id][pod] = struct{}{} + } + for id, pods := range coreToPods { + require.Equal(t, 1, len(pods), + "%s on %s: %s is attributed to %d pods %v", + md.Name, node, id, len(pods), sortedKeys(pods)) + } + } + }) + } +} + +// TestMultiRuntimeTagNotFlattened verifies that the runtime tag carries a real +// runtime value rather than the aggregation sentinel, so per-runtime attribution +// is preserved. +func TestMultiRuntimeTagNotFlattened(t *testing.T) { + t.Parallel() + for _, metricName := range neuronMetricNamesList { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (no Neuron nodes?)", metricName) + + for _, r := range results { + r := r + tag, ok := r.Labels.Resource[runtimeTagResourceKey] + if !ok { + continue + } + require.NotEqual(t, runtimeTagFlattenedSentinel, tag, + "%s carries @resource.%s=%q: the runtime dimension was aggregated away "+ + "in-agent, losing per-runtime attribution", + metricName, runtimeTagResourceKey, tag) + } + }) + } +} + +// TestMultiRuntimeTagAbsentFromDatapoint verifies that the runtime tag is a +// non-empty resource attribute and not a datapoint attribute. +func TestMultiRuntimeTagAbsentFromDatapoint(t *testing.T) { + t.Parallel() + for _, md := range neuronCoreLevelMetrics { + md := md + t.Run(md.Name, func(t *testing.T) { + t.Parallel() + results, nodes := multiRuntimeResults(t, md.Name) + + for _, node := range nodes { + node := node + for _, r := range resultsForNode(results, node) { + r := r + _, hasDatapointTag := r.Labels.Datapoint["runtime_tag"] + require.False(t, hasDatapointTag, + "%s on %s: runtime_tag is still a datapoint attribute (%q)", + md.Name, node, r.Labels.Datapoint["runtime_tag"]) + + tag, hasResourceTag := r.Labels.Resource[runtimeTagResourceKey] + require.True(t, hasResourceTag, + "%s on %s: missing @resource.%s", md.Name, node, runtimeTagResourceKey) + require.NotEmpty(t, tag, + "%s on %s: @resource.%s is empty", md.Name, node, runtimeTagResourceKey) + } + } + }) + } +} diff --git a/test/otel/neuron/neuron_test.go b/test/otel/neuron/neuron_test.go index 3774b54b5..abe2e578e 100644 --- a/test/otel/neuron/neuron_test.go +++ b/test/otel/neuron/neuron_test.go @@ -207,7 +207,7 @@ func TestNeuronRuntimeTagInResourceScope(t *testing.T) { "No correlated %s results to check runtime tag", md.Name) for _, r := range correlated { r := r - tag, hasTag := r.Labels.Resource["aws.neuron.runtime.tag"] + tag, hasTag := r.Labels.Resource[runtimeTagResourceKey] require.True(t, hasTag, "%s correlated result missing @resource.aws.neuron.runtime.tag (pod: %s)", md.Name, r.Labels.Resource["k8s.pod.name"])