diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index 8ed76948..ff0e2f1b 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -7,6 +7,19 @@ SPDX-License-Identifier: Apache-2.0 A Go service that exposes this machine's hardware inventory (GPUs, CPU, physical memory) over a small HTTP API at `/v1/node-info`. It advertises nothing over mDNS itself — its parent (the broker) registers its `ni` port with the `nvpair-node-scanner` discovery daemon, which folds it into this node's single `_nvpair-node` record and fetches `/v1/node-info` to enrich the node for peers. +## Linux AMD telemetry + +AMD adapters use PCI identity and the amdgpu `gpu_busy_percent` sysfs counter. +This requires neither ROCm nor CUDA. Missing, unreadable, malformed, or out of +range samples are unavailable. Valid samples, including idle samples, refresh +the existing `telemetryValid` and `msSince` freshness contract. The existing +wire format omits a zero utilization value, so inspect freshness as well. + +AMD memory fields remain omitted. On Strix Halo, reserved framebuffer VRAM, +GTT capacity, the HSA GPU-visible pool, and system MemAvailable are distinct +and overlapping measurements. This change does not label system RAM or GTT +as dedicated VRAM. NVIDIA collection remains enabled on mixed-vendor hosts. + ## Communication Two surfaces: diff --git a/services/nvpair-node-info/gpu_linux.go b/services/nvpair-node-info/gpu_linux.go index fdc2fa97..90a8e87f 100644 --- a/services/nvpair-node-info/gpu_linux.go +++ b/services/nvpair-node-info/gpu_linux.go @@ -8,7 +8,9 @@ package main import ( "context" "log" + "os" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -22,13 +24,15 @@ import ( // collector from blocking indefinitely. const nvidiaSmiTimeout = 3 * time.Second +const amdPCIRoot = "/sys/bus/pci/devices" + // detectGPUs enumerates GPUs on Linux. It prefers nvidia-smi, which yields the // marketing name, total VRAM, and a stable per-GPU UUID we reuse as the join // key (statsKey) against the dynamic stats collector's snapshot. When // nvidia-smi is absent — no NVIDIA driver, or an AMD/Intel-only host — it falls -// back to ghw, which reports adapter names but no VRAM and no join key, so -// those hosts list their GPUs without dynamic VRAM/utilization (matching the -// pre-existing non-Windows behavior). +// back to ghw. AMD adapters receive a PCI-derived join key for dynamic +// utilization, while other adapters list their names without dynamic VRAM or +// utilization (matching the pre-existing non-Windows behavior). // // On unified-memory architectures (UMA, e.g. Grace-Blackwell / DGX Spark) // nvidia-smi reports [N/A] for memory.total because the GPU shares system @@ -45,6 +49,12 @@ func detectGPUs() []GPUInfo { } } } + // Preserve NVIDIA records and include AMD adapters on mixed hosts. + for _, gpu := range detectGPUsGHW() { + if strings.HasPrefix(gpu.statsKey, "amd:") { + gpus = append(gpus, gpu) + } + } return gpus } } @@ -53,7 +63,8 @@ func detectGPUs() []GPUInfo { // detectGPUsGHW is the ghw-based fallback, identical in spirit to the // non-Windows/non-Linux path in gpu_other.go: enumerate display adapters and -// return names only (VramBytes stays 0, statsKey stays empty). +// return adapter names. AMD adapters also receive PCI-derived stats keys for +// dynamic utilization; VramBytes stays 0 for every ghw-derived adapter. func detectGPUsGHW() []GPUInfo { gpu, err := ghw.GPU() if err != nil { @@ -66,11 +77,56 @@ func detectGPUsGHW() []GPUInfo { if card.DeviceInfo != nil && card.DeviceInfo.Product != nil { name = card.DeviceInfo.Product.Name } - gpus = append(gpus, GPUInfo{Name: name}) + gpus = append(gpus, GPUInfo{Name: name, statsKey: amdStatsKey(amdPCIRoot, card.Address)}) } return gpus } +// amdStatsKey identifies AMD adapters by PCI address, independently of DRM +// card numbering. The root argument permits tests without real hardware. +func amdStatsKey(root, address string) string { + if !strings.Contains(address, ":") || filepath.Base(address) != address { + return "" + } + if strings.Count(address, ":") == 1 { + address = "0000:" + address + } + data, err := os.ReadFile(filepath.Join(root, address, "vendor")) + if err != nil || strings.TrimSpace(string(data)) != "0x1002" { + return "" + } + return "amd:" + address +} + +// decodeAMDUtilization reads the amdgpu driver counter without a ROCm library, +// subprocess or elevated privilege. Missing or invalid counters stay absent. +// Memory is deliberately not mapped to VRAM: Strix Halo's GTT, reserved VRAM, +// HSA pool and MemAvailable have different semantics and overlap. +func decodeAMDUtilization(root string, out map[string]gpuStat) bool { + entries, err := os.ReadDir(root) + if err != nil { + return false + } + valid := false + for _, entry := range entries { + key := amdStatsKey(root, entry.Name()) + if key == "" { + continue + } + data, err := os.ReadFile(filepath.Join(root, entry.Name(), "gpu_busy_percent")) + if err != nil { + continue + } + pct, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 32) + if err != nil || pct > 100 { + continue + } + out[key] = gpuStat{UtilizationPct: uint32(pct)} + valid = true + } + return valid +} + // nvidiaSmiCSV runs `nvidia-smi --query-gpu= --format=csv,noheader,nounits` // and returns raw stdout. The caller parses the comma-separated rows. A missing // binary (not on PATH) surfaces as an exec error, which callers treat as "no diff --git a/services/nvpair-node-info/gpu_linux_test.go b/services/nvpair-node-info/gpu_linux_test.go new file mode 100644 index 00000000..97aa72cd --- /dev/null +++ b/services/nvpair-node-info/gpu_linux_test.go @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestAMDUtilizationAvailability(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "0000:c5:00.0") + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + write := func(name, value string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + write("vendor", "0x1002\n") + if got := amdStatsKey(root, "c5:00.0"); got != "amd:0000:c5:00.0" { + t.Fatal(got) + } + for _, value := range []string{"0", "83", "100", "101", "-1", "N/A"} { + write("gpu_busy_percent", value) + out := map[string]gpuStat{"NVIDIA-existing": {UtilizationPct: 7}} + valid := decodeAMDUtilization(root, out) + want := value == "0" || value == "83" || value == "100" + if valid != want { + t.Fatalf("%q valid=%v", value, valid) + } + _, exists := out["amd:0000:c5:00.0"] + if exists != want { + t.Fatalf("%q fabricated or lost sample", value) + } + if out["NVIDIA-existing"].UtilizationPct != 7 { + t.Fatal("NVIDIA overwritten") + } + } + if err := os.Remove(filepath.Join(dir, "gpu_busy_percent")); err != nil { + t.Fatal(err) + } + if decodeAMDUtilization(root, map[string]gpuStat{}) { + t.Fatal("missing counter marked valid") + } + write("vendor", "0x8086") + write("gpu_busy_percent", "70") + if decodeAMDUtilization(root, map[string]gpuStat{}) { + t.Fatal("non-AMD adapter accepted") + } +} diff --git a/services/nvpair-node-info/stats_linux.go b/services/nvpair-node-info/stats_linux.go index 30b6f4df..727b7d1e 100644 --- a/services/nvpair-node-info/stats_linux.go +++ b/services/nvpair-node-info/stats_linux.go @@ -143,7 +143,9 @@ func (c *statsCollector) decodeSnapshot() *statsSnapshot { gpu := make(map[string]gpuStat) sampledAt := time.Time{} - if c.decodeGPU(gpu) { + nvidiaValid := c.decodeGPU(gpu) + amdValid := decodeAMDUtilization(amdPCIRoot, gpu) + if nvidiaValid || amdValid { sampledAt = time.Now() } applyGPUStats(previous, snap, gpu, sampledAt) @@ -161,7 +163,7 @@ func (c *statsCollector) decodeGPU(out map[string]gpuStat) bool { csv, err := nvidiaSmiCSV("uuid,utilization.gpu,memory.used") if err != nil { if c.nvidiaUnavailable.CompareAndSwap(false, true) { - slog.Warn("nvidia-smi unavailable; GPU utilization / dedicated VRAM-used will not be reported", + slog.Debug("nvidia-smi unavailable; NVIDIA metrics unavailable; other GPU sources remain enabled", "err", err) } return false diff --git a/services/versions.json b/services/versions.json index 29d8c230..a76c03d7 100644 --- a/services/versions.json +++ b/services/versions.json @@ -5,7 +5,7 @@ "components": { "ollama-proxy": "0.26.2", "lmstudio-proxy": "0.16.2", - "nvpair-node-info": "0.13.3", + "nvpair-node-info": "0.14.0", "nvpair-node-scanner": "0.20.3", "nvpair-manual-nodes": "0.11.1", "nvpair-workload-manager": "0.13.3",