From 1da1f636a1468949fac98552ced9300ec1133728 Mon Sep 17 00:00:00 2001 From: David Bourdeau <42303109+dbourdea@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:07:53 -0700 Subject: [PATCH 1/2] feat(node-info): report Linux AMD GPU utilization Signed-off-by: David Bourdeau <42303109+dbourdea@users.noreply.github.com> --- services/nvpair-node-info/README.md | 13 +++++ services/nvpair-node-info/amd_linux.go | 60 +++++++++++++++++++++ services/nvpair-node-info/amd_linux_test.go | 57 ++++++++++++++++++++ services/nvpair-node-info/gpu_linux.go | 8 ++- services/nvpair-node-info/stats_linux.go | 6 ++- services/versions.json | 2 +- 6 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 services/nvpair-node-info/amd_linux.go create mode 100644 services/nvpair-node-info/amd_linux_test.go diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index 8ed76948..ff0e2f1b 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -7,6 +7,19 @@ SPDX-License-Identifier: Apache-2.0 A Go service that exposes this machine's hardware inventory (GPUs, CPU, physical memory) over a small HTTP API at `/v1/node-info`. It advertises nothing over mDNS itself — its parent (the broker) registers its `ni` port with the `nvpair-node-scanner` discovery daemon, which folds it into this node's single `_nvpair-node` record and fetches `/v1/node-info` to enrich the node for peers. +## Linux AMD telemetry + +AMD adapters use PCI identity and the amdgpu `gpu_busy_percent` sysfs counter. +This requires neither ROCm nor CUDA. Missing, unreadable, malformed, or out of +range samples are unavailable. Valid samples, including idle samples, refresh +the existing `telemetryValid` and `msSince` freshness contract. The existing +wire format omits a zero utilization value, so inspect freshness as well. + +AMD memory fields remain omitted. On Strix Halo, reserved framebuffer VRAM, +GTT capacity, the HSA GPU-visible pool, and system MemAvailable are distinct +and overlapping measurements. This change does not label system RAM or GTT +as dedicated VRAM. NVIDIA collection remains enabled on mixed-vendor hosts. + ## Communication Two surfaces: diff --git a/services/nvpair-node-info/amd_linux.go b/services/nvpair-node-info/amd_linux.go new file mode 100644 index 00000000..6e245867 --- /dev/null +++ b/services/nvpair-node-info/amd_linux.go @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "os" + "path/filepath" + "strconv" + "strings" +) + +const amdPCIRoot = "/sys/bus/pci/devices" + +// amdStatsKey identifies AMD adapters by PCI address, independently of DRM +// card numbering. The root argument permits tests without real hardware. +func amdStatsKey(root, address string) string { + if !strings.Contains(address, ":") || filepath.Base(address) != address { + return "" + } + if strings.Count(address, ":") == 1 { + address = "0000:" + address + } + data, err := os.ReadFile(filepath.Join(root, address, "vendor")) + if err != nil || strings.TrimSpace(string(data)) != "0x1002" { + return "" + } + return "amd:" + address +} + +// decodeAMDUtilization reads the amdgpu driver counter without a ROCm library, +// subprocess or elevated privilege. Missing or invalid counters stay absent. +// Memory is deliberately not mapped to VRAM: Strix Halo's GTT, reserved VRAM, +// HSA pool and MemAvailable have different semantics and overlap. +func decodeAMDUtilization(root string, out map[string]gpuStat) bool { + entries, err := os.ReadDir(root) + if err != nil { + return false + } + valid := false + for _, entry := range entries { + key := amdStatsKey(root, entry.Name()) + if key == "" { + continue + } + data, err := os.ReadFile(filepath.Join(root, entry.Name(), "gpu_busy_percent")) + if err != nil { + continue + } + pct, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 32) + if err != nil || pct > 100 { + continue + } + out[key] = gpuStat{UtilizationPct: uint32(pct)} + valid = true + } + return valid +} diff --git a/services/nvpair-node-info/amd_linux_test.go b/services/nvpair-node-info/amd_linux_test.go new file mode 100644 index 00000000..97aa72cd --- /dev/null +++ b/services/nvpair-node-info/amd_linux_test.go @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestAMDUtilizationAvailability(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "0000:c5:00.0") + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + write := func(name, value string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + write("vendor", "0x1002\n") + if got := amdStatsKey(root, "c5:00.0"); got != "amd:0000:c5:00.0" { + t.Fatal(got) + } + for _, value := range []string{"0", "83", "100", "101", "-1", "N/A"} { + write("gpu_busy_percent", value) + out := map[string]gpuStat{"NVIDIA-existing": {UtilizationPct: 7}} + valid := decodeAMDUtilization(root, out) + want := value == "0" || value == "83" || value == "100" + if valid != want { + t.Fatalf("%q valid=%v", value, valid) + } + _, exists := out["amd:0000:c5:00.0"] + if exists != want { + t.Fatalf("%q fabricated or lost sample", value) + } + if out["NVIDIA-existing"].UtilizationPct != 7 { + t.Fatal("NVIDIA overwritten") + } + } + if err := os.Remove(filepath.Join(dir, "gpu_busy_percent")); err != nil { + t.Fatal(err) + } + if decodeAMDUtilization(root, map[string]gpuStat{}) { + t.Fatal("missing counter marked valid") + } + write("vendor", "0x8086") + write("gpu_busy_percent", "70") + if decodeAMDUtilization(root, map[string]gpuStat{}) { + t.Fatal("non-AMD adapter accepted") + } +} diff --git a/services/nvpair-node-info/gpu_linux.go b/services/nvpair-node-info/gpu_linux.go index fdc2fa97..b8abc5b1 100644 --- a/services/nvpair-node-info/gpu_linux.go +++ b/services/nvpair-node-info/gpu_linux.go @@ -45,6 +45,12 @@ func detectGPUs() []GPUInfo { } } } + // Preserve NVIDIA records and include AMD adapters on mixed hosts. + for _, gpu := range detectGPUsGHW() { + if strings.HasPrefix(gpu.statsKey, "amd:") { + gpus = append(gpus, gpu) + } + } return gpus } } @@ -66,7 +72,7 @@ func detectGPUsGHW() []GPUInfo { if card.DeviceInfo != nil && card.DeviceInfo.Product != nil { name = card.DeviceInfo.Product.Name } - gpus = append(gpus, GPUInfo{Name: name}) + gpus = append(gpus, GPUInfo{Name: name, statsKey: amdStatsKey(amdPCIRoot, card.Address)}) } return gpus } diff --git a/services/nvpair-node-info/stats_linux.go b/services/nvpair-node-info/stats_linux.go index 30b6f4df..727b7d1e 100644 --- a/services/nvpair-node-info/stats_linux.go +++ b/services/nvpair-node-info/stats_linux.go @@ -143,7 +143,9 @@ func (c *statsCollector) decodeSnapshot() *statsSnapshot { gpu := make(map[string]gpuStat) sampledAt := time.Time{} - if c.decodeGPU(gpu) { + nvidiaValid := c.decodeGPU(gpu) + amdValid := decodeAMDUtilization(amdPCIRoot, gpu) + if nvidiaValid || amdValid { sampledAt = time.Now() } applyGPUStats(previous, snap, gpu, sampledAt) @@ -161,7 +163,7 @@ func (c *statsCollector) decodeGPU(out map[string]gpuStat) bool { csv, err := nvidiaSmiCSV("uuid,utilization.gpu,memory.used") if err != nil { if c.nvidiaUnavailable.CompareAndSwap(false, true) { - slog.Warn("nvidia-smi unavailable; GPU utilization / dedicated VRAM-used will not be reported", + slog.Debug("nvidia-smi unavailable; NVIDIA metrics unavailable; other GPU sources remain enabled", "err", err) } return false diff --git a/services/versions.json b/services/versions.json index 29d8c230..a76c03d7 100644 --- a/services/versions.json +++ b/services/versions.json @@ -5,7 +5,7 @@ "components": { "ollama-proxy": "0.26.2", "lmstudio-proxy": "0.16.2", - "nvpair-node-info": "0.13.3", + "nvpair-node-info": "0.14.0", "nvpair-node-scanner": "0.20.3", "nvpair-manual-nodes": "0.11.1", "nvpair-workload-manager": "0.13.3", From 5279297e8597b10479686ca337e42677879f0779 Mon Sep 17 00:00:00 2001 From: David Bourdeau <42303109+dbourdea@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:52:08 -0700 Subject: [PATCH 2/2] refactor(node-info): co-locate Linux AMD GPU telemetry Signed-off-by: David Bourdeau <42303109+dbourdea@users.noreply.github.com> --- services/nvpair-node-info/amd_linux.go | 60 ------------------- services/nvpair-node-info/gpu_linux.go | 58 ++++++++++++++++-- .../{amd_linux_test.go => gpu_linux_test.go} | 0 3 files changed, 54 insertions(+), 64 deletions(-) delete mode 100644 services/nvpair-node-info/amd_linux.go rename services/nvpair-node-info/{amd_linux_test.go => gpu_linux_test.go} (100%) diff --git a/services/nvpair-node-info/amd_linux.go b/services/nvpair-node-info/amd_linux.go deleted file mode 100644 index 6e245867..00000000 --- a/services/nvpair-node-info/amd_linux.go +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//go:build linux - -package main - -import ( - "os" - "path/filepath" - "strconv" - "strings" -) - -const amdPCIRoot = "/sys/bus/pci/devices" - -// amdStatsKey identifies AMD adapters by PCI address, independently of DRM -// card numbering. The root argument permits tests without real hardware. -func amdStatsKey(root, address string) string { - if !strings.Contains(address, ":") || filepath.Base(address) != address { - return "" - } - if strings.Count(address, ":") == 1 { - address = "0000:" + address - } - data, err := os.ReadFile(filepath.Join(root, address, "vendor")) - if err != nil || strings.TrimSpace(string(data)) != "0x1002" { - return "" - } - return "amd:" + address -} - -// decodeAMDUtilization reads the amdgpu driver counter without a ROCm library, -// subprocess or elevated privilege. Missing or invalid counters stay absent. -// Memory is deliberately not mapped to VRAM: Strix Halo's GTT, reserved VRAM, -// HSA pool and MemAvailable have different semantics and overlap. -func decodeAMDUtilization(root string, out map[string]gpuStat) bool { - entries, err := os.ReadDir(root) - if err != nil { - return false - } - valid := false - for _, entry := range entries { - key := amdStatsKey(root, entry.Name()) - if key == "" { - continue - } - data, err := os.ReadFile(filepath.Join(root, entry.Name(), "gpu_busy_percent")) - if err != nil { - continue - } - pct, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 32) - if err != nil || pct > 100 { - continue - } - out[key] = gpuStat{UtilizationPct: uint32(pct)} - valid = true - } - return valid -} diff --git a/services/nvpair-node-info/gpu_linux.go b/services/nvpair-node-info/gpu_linux.go index b8abc5b1..90a8e87f 100644 --- a/services/nvpair-node-info/gpu_linux.go +++ b/services/nvpair-node-info/gpu_linux.go @@ -8,7 +8,9 @@ package main import ( "context" "log" + "os" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -22,13 +24,15 @@ import ( // collector from blocking indefinitely. const nvidiaSmiTimeout = 3 * time.Second +const amdPCIRoot = "/sys/bus/pci/devices" + // detectGPUs enumerates GPUs on Linux. It prefers nvidia-smi, which yields the // marketing name, total VRAM, and a stable per-GPU UUID we reuse as the join // key (statsKey) against the dynamic stats collector's snapshot. When // nvidia-smi is absent — no NVIDIA driver, or an AMD/Intel-only host — it falls -// back to ghw, which reports adapter names but no VRAM and no join key, so -// those hosts list their GPUs without dynamic VRAM/utilization (matching the -// pre-existing non-Windows behavior). +// back to ghw. AMD adapters receive a PCI-derived join key for dynamic +// utilization, while other adapters list their names without dynamic VRAM or +// utilization (matching the pre-existing non-Windows behavior). // // On unified-memory architectures (UMA, e.g. Grace-Blackwell / DGX Spark) // nvidia-smi reports [N/A] for memory.total because the GPU shares system @@ -59,7 +63,8 @@ func detectGPUs() []GPUInfo { // detectGPUsGHW is the ghw-based fallback, identical in spirit to the // non-Windows/non-Linux path in gpu_other.go: enumerate display adapters and -// return names only (VramBytes stays 0, statsKey stays empty). +// return adapter names. AMD adapters also receive PCI-derived stats keys for +// dynamic utilization; VramBytes stays 0 for every ghw-derived adapter. func detectGPUsGHW() []GPUInfo { gpu, err := ghw.GPU() if err != nil { @@ -77,6 +82,51 @@ func detectGPUsGHW() []GPUInfo { return gpus } +// amdStatsKey identifies AMD adapters by PCI address, independently of DRM +// card numbering. The root argument permits tests without real hardware. +func amdStatsKey(root, address string) string { + if !strings.Contains(address, ":") || filepath.Base(address) != address { + return "" + } + if strings.Count(address, ":") == 1 { + address = "0000:" + address + } + data, err := os.ReadFile(filepath.Join(root, address, "vendor")) + if err != nil || strings.TrimSpace(string(data)) != "0x1002" { + return "" + } + return "amd:" + address +} + +// decodeAMDUtilization reads the amdgpu driver counter without a ROCm library, +// subprocess or elevated privilege. Missing or invalid counters stay absent. +// Memory is deliberately not mapped to VRAM: Strix Halo's GTT, reserved VRAM, +// HSA pool and MemAvailable have different semantics and overlap. +func decodeAMDUtilization(root string, out map[string]gpuStat) bool { + entries, err := os.ReadDir(root) + if err != nil { + return false + } + valid := false + for _, entry := range entries { + key := amdStatsKey(root, entry.Name()) + if key == "" { + continue + } + data, err := os.ReadFile(filepath.Join(root, entry.Name(), "gpu_busy_percent")) + if err != nil { + continue + } + pct, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 32) + if err != nil || pct > 100 { + continue + } + out[key] = gpuStat{UtilizationPct: uint32(pct)} + valid = true + } + return valid +} + // nvidiaSmiCSV runs `nvidia-smi --query-gpu= --format=csv,noheader,nounits` // and returns raw stdout. The caller parses the comma-separated rows. A missing // binary (not on PATH) surfaces as an exec error, which callers treat as "no diff --git a/services/nvpair-node-info/amd_linux_test.go b/services/nvpair-node-info/gpu_linux_test.go similarity index 100% rename from services/nvpair-node-info/amd_linux_test.go rename to services/nvpair-node-info/gpu_linux_test.go