Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions services/nvpair-node-info/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,19 @@ SPDX-License-Identifier: Apache-2.0

A Go service that exposes this machine's hardware inventory (GPUs, CPU, physical memory) over a small HTTP API at `/v1/node-info`. It advertises nothing over mDNS itself — its parent (the broker) registers its `ni` port with the `nvpair-node-scanner` discovery daemon, which folds it into this node's single `_nvpair-node` record and fetches `/v1/node-info` to enrich the node for peers.

## Linux AMD telemetry

AMD adapters use PCI identity and the amdgpu `gpu_busy_percent` sysfs counter.
This requires neither ROCm nor CUDA. Missing, unreadable, malformed, or out of
range samples are unavailable. Valid samples, including idle samples, refresh
the existing `telemetryValid` and `msSince` freshness contract. The existing
wire format omits a zero utilization value, so inspect freshness as well.

AMD memory fields remain omitted. On Strix Halo, reserved framebuffer VRAM,
GTT capacity, the HSA GPU-visible pool, and system MemAvailable are distinct
and overlapping measurements. This change does not label system RAM or GTT
as dedicated VRAM. NVIDIA collection remains enabled on mixed-vendor hosts.

## Communication

Two surfaces:
Expand Down
66 changes: 61 additions & 5 deletions services/nvpair-node-info/gpu_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ package main
import (
"context"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -22,13 +24,15 @@ import (
// collector from blocking indefinitely.
const nvidiaSmiTimeout = 3 * time.Second

const amdPCIRoot = "/sys/bus/pci/devices"

// detectGPUs enumerates GPUs on Linux. It prefers nvidia-smi, which yields the
// marketing name, total VRAM, and a stable per-GPU UUID we reuse as the join
// key (statsKey) against the dynamic stats collector's snapshot. When
// nvidia-smi is absent — no NVIDIA driver, or an AMD/Intel-only host — it falls
// back to ghw, which reports adapter names but no VRAM and no join key, so
// those hosts list their GPUs without dynamic VRAM/utilization (matching the
// pre-existing non-Windows behavior).
// back to ghw. AMD adapters receive a PCI-derived join key for dynamic
// utilization, while other adapters list their names without dynamic VRAM or
// utilization (matching the pre-existing non-Windows behavior).
//
// On unified-memory architectures (UMA, e.g. Grace-Blackwell / DGX Spark)
// nvidia-smi reports [N/A] for memory.total because the GPU shares system
Expand All @@ -45,6 +49,12 @@ func detectGPUs() []GPUInfo {
}
}
}
// Preserve NVIDIA records and include AMD adapters on mixed hosts.
for _, gpu := range detectGPUsGHW() {
if strings.HasPrefix(gpu.statsKey, "amd:") {
gpus = append(gpus, gpu)
}
}
return gpus
}
}
Expand All @@ -53,7 +63,8 @@ func detectGPUs() []GPUInfo {

// detectGPUsGHW is the ghw-based fallback, identical in spirit to the
// non-Windows/non-Linux path in gpu_other.go: enumerate display adapters and
// return names only (VramBytes stays 0, statsKey stays empty).
// return adapter names. AMD adapters also receive PCI-derived stats keys for
// dynamic utilization; VramBytes stays 0 for every ghw-derived adapter.
func detectGPUsGHW() []GPUInfo {
gpu, err := ghw.GPU()
if err != nil {
Expand All @@ -66,11 +77,56 @@ func detectGPUsGHW() []GPUInfo {
if card.DeviceInfo != nil && card.DeviceInfo.Product != nil {
name = card.DeviceInfo.Product.Name
}
gpus = append(gpus, GPUInfo{Name: name})
gpus = append(gpus, GPUInfo{Name: name, statsKey: amdStatsKey(amdPCIRoot, card.Address)})
}
return gpus
}

// amdStatsKey identifies AMD adapters by PCI address, independently of DRM
// card numbering. The root argument permits tests without real hardware.
func amdStatsKey(root, address string) string {
if !strings.Contains(address, ":") || filepath.Base(address) != address {
return ""
}
if strings.Count(address, ":") == 1 {
address = "0000:" + address
}
data, err := os.ReadFile(filepath.Join(root, address, "vendor"))
if err != nil || strings.TrimSpace(string(data)) != "0x1002" {
return ""
}
return "amd:" + address
}

// decodeAMDUtilization reads the amdgpu driver counter without a ROCm library,
// subprocess or elevated privilege. Missing or invalid counters stay absent.
// Memory is deliberately not mapped to VRAM: Strix Halo's GTT, reserved VRAM,
// HSA pool and MemAvailable have different semantics and overlap.
func decodeAMDUtilization(root string, out map[string]gpuStat) bool {
entries, err := os.ReadDir(root)
if err != nil {
return false
}
valid := false
for _, entry := range entries {
key := amdStatsKey(root, entry.Name())
if key == "" {
continue
}
data, err := os.ReadFile(filepath.Join(root, entry.Name(), "gpu_busy_percent"))
if err != nil {
continue
}
pct, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 32)
if err != nil || pct > 100 {
continue
}
out[key] = gpuStat{UtilizationPct: uint32(pct)}
valid = true
}
return valid
}

// nvidiaSmiCSV runs `nvidia-smi --query-gpu=<fields> --format=csv,noheader,nounits`
// and returns raw stdout. The caller parses the comma-separated rows. A missing
// binary (not on PATH) surfaces as an exec error, which callers treat as "no
Expand Down
57 changes: 57 additions & 0 deletions services/nvpair-node-info/gpu_linux_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//go:build linux

package main

import (
"os"
"path/filepath"
"testing"
)

func TestAMDUtilizationAvailability(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "0000:c5:00.0")
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
write := func(name, value string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
t.Fatal(err)
}
}
write("vendor", "0x1002\n")
if got := amdStatsKey(root, "c5:00.0"); got != "amd:0000:c5:00.0" {
t.Fatal(got)
}
for _, value := range []string{"0", "83", "100", "101", "-1", "N/A"} {
write("gpu_busy_percent", value)
out := map[string]gpuStat{"NVIDIA-existing": {UtilizationPct: 7}}
valid := decodeAMDUtilization(root, out)
want := value == "0" || value == "83" || value == "100"
if valid != want {
t.Fatalf("%q valid=%v", value, valid)
}
_, exists := out["amd:0000:c5:00.0"]
if exists != want {
t.Fatalf("%q fabricated or lost sample", value)
}
if out["NVIDIA-existing"].UtilizationPct != 7 {
t.Fatal("NVIDIA overwritten")
}
}
if err := os.Remove(filepath.Join(dir, "gpu_busy_percent")); err != nil {
t.Fatal(err)
}
if decodeAMDUtilization(root, map[string]gpuStat{}) {
t.Fatal("missing counter marked valid")
}
write("vendor", "0x8086")
write("gpu_busy_percent", "70")
if decodeAMDUtilization(root, map[string]gpuStat{}) {
t.Fatal("non-AMD adapter accepted")
}
}
6 changes: 4 additions & 2 deletions services/nvpair-node-info/stats_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,9 @@ func (c *statsCollector) decodeSnapshot() *statsSnapshot {

gpu := make(map[string]gpuStat)
sampledAt := time.Time{}
if c.decodeGPU(gpu) {
nvidiaValid := c.decodeGPU(gpu)
amdValid := decodeAMDUtilization(amdPCIRoot, gpu)
if nvidiaValid || amdValid {
sampledAt = time.Now()
}
applyGPUStats(previous, snap, gpu, sampledAt)
Expand All @@ -161,7 +163,7 @@ func (c *statsCollector) decodeGPU(out map[string]gpuStat) bool {
csv, err := nvidiaSmiCSV("uuid,utilization.gpu,memory.used")
if err != nil {
if c.nvidiaUnavailable.CompareAndSwap(false, true) {
slog.Warn("nvidia-smi unavailable; GPU utilization / dedicated VRAM-used will not be reported",
slog.Debug("nvidia-smi unavailable; NVIDIA metrics unavailable; other GPU sources remain enabled",
"err", err)
}
return false
Expand Down
2 changes: 1 addition & 1 deletion services/versions.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"components": {
"ollama-proxy": "0.26.2",
"lmstudio-proxy": "0.16.2",
"nvpair-node-info": "0.13.3",
"nvpair-node-info": "0.14.0",
"nvpair-node-scanner": "0.20.3",
"nvpair-manual-nodes": "0.11.1",
"nvpair-workload-manager": "0.13.3",
Expand Down