From 77bcc3189972a899903e09461a793b95d7a39bea Mon Sep 17 00:00:00 2001 From: elyerinfox Date: Mon, 14 Sep 2026 06:04:00 +0000 Subject: [PATCH 1/3] Detect Intel Arc GPUs on Linux Adds an Intel detector (xpu-smi when installed, sysfs fallback) that runs in parallel with the existing NVIDIA path, so mixed hosts report every adapter. Signed-off-by: elyerinfox --- README.md | 8 + services/nvpair-node-info/README.md | 7 +- services/nvpair-node-info/gpu_intel_linux.go | 382 +++++++++++++++ .../nvpair-node-info/gpu_intel_linux_test.go | 283 +++++++++++ services/nvpair-node-info/gpu_linux.go | 72 ++- .../nvpair-node-info/stats_intel_linux.go | 458 ++++++++++++++++++ services/nvpair-node-info/stats_linux.go | 44 +- 7 files changed, 1223 insertions(+), 31 deletions(-) create mode 100644 services/nvpair-node-info/gpu_intel_linux.go create mode 100644 services/nvpair-node-info/gpu_intel_linux_test.go create mode 100644 services/nvpair-node-info/stats_intel_linux.go diff --git a/README.md b/README.md index 0f0a7242..b8d5550f 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ before assuming a node can serve a model. A node only becomes a candidate for a request once it is actually running a compatible engine, and PAIR prefers the nodes it already knows hold the model. +**GPU inventory reporting** covers NVIDIA, Intel (Arc dGPU and iGPU), AMD, and +Apple Silicon adapters. On Windows every vendor reports through the same DXGI/PDH +pipeline. On Linux, NVIDIA data comes from `nvidia-smi`, Intel data comes from +`xpu-smi` when installed (falling back to a `/sys/class/drm` sysfs walk so Arc +adapters are still identified without extra tooling), and other vendors fall +back to names only. See [services/nvpair-node-info/README.md](services/nvpair-node-info/README.md#platform-notes) +for the full source-of-truth table. + ## Quick start Download a released build and use the desktop application. That is the path we diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index 8ed76948..020f1113 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -91,8 +91,11 @@ The service no longer advertises itself over mDNS. Its parent (the broker) regis ## Platform Notes -- **Windows** (first-class): GPU inventory comes from DXGI (vendor-agnostic, includes VRAM). Dynamic CPU / VRAM-used / utilization / memory-used numbers come from a persistent PDH query plus `GlobalMemoryStatusEx`. -- **Linux** (first-class): NVIDIA GPU inventory, dedicated VRAM usage, and utilization come from `nvidia-smi`; CPU and system-memory usage come from `/proc`. Unified-memory GPUs use the `/proc/meminfo` system-memory snapshot even when dynamic `nvidia-smi` collection is unavailable. Non-NVIDIA adapters fall back to names from `ghw` without dynamic GPU stats. +- **Windows** (first-class): GPU inventory comes from DXGI (vendor-agnostic — NVIDIA, AMD, Intel Arc, and iGPUs enumerate through the same path, including VRAM). Dynamic CPU / VRAM-used / utilization / memory-used numbers come from a persistent PDH query plus `GlobalMemoryStatusEx`. Intel Arc adapters are handled through this same pipeline with no extra software required. +- **Linux** (first-class): Multiple vendor detectors run in parallel and their results are combined, so a mixed host reports every adapter: + - **NVIDIA**: inventory, dedicated VRAM, and utilization come from `nvidia-smi`. Unified-memory GPUs (Grace-Blackwell / DGX Spark) use the `/proc/meminfo` system-memory snapshot even when dynamic `nvidia-smi` collection is unavailable. + - **Intel (Arc dGPU and iGPU)**: identity comes from `xpu-smi discovery --json` when installed, otherwise from a sysfs walk of `/sys/class/drm/card*` filtered by PCI vendor `0x8086`. Dedicated VRAM totals come from `xpu-smi` or the DRM `mem_info_vram_total` attribute (Arc dGPU only). Dynamic memory-used comes from `xpu-smi dump` or the DRM `mem_info_vram_used` attribute. Utilization requires `xpu-smi`; without it the field is omitted (Intel does not expose a stable busy-percent under DRM). Adapters are joined across sources by PCI BDF (statsKey prefix `intel-pci-`). + - CPU and system-memory usage come from `/proc`. Adapters from unsupported vendors fall back to names from `ghw` without dynamic GPU stats. - **macOS**: CPU and system-memory usage come from Mach through gopsutil's purego bindings. GPU identity, mapped memory, and utilization come from the built-in, unprivileged `/usr/sbin/ioreg` command's `IOAccelerator` `PerformanceStatistics`; no sudo or private framework binding is required. Apple Silicon is supported directly. Intel/AMD fields are best-effort when their drivers expose the same dedicated-memory counters. The performance keys are undocumented and may change across macOS releases; a missing or changed key leaves only that metric out and does not stop CPU or memory collection. - **Other platforms**: GPU names come from `ghw`; VRAM and dynamic stats are not reported. diff --git a/services/nvpair-node-info/gpu_intel_linux.go b/services/nvpair-node-info/gpu_intel_linux.go new file mode 100644 index 00000000..531ffee6 --- /dev/null +++ b/services/nvpair-node-info/gpu_intel_linux.go @@ -0,0 +1,382 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "context" + "encoding/json" + "log/slog" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/jaypipes/ghw" +) + +// Intel discrete and integrated GPU detection on Linux. +// +// Data sources, in preference order: +// +// 1. xpu-smi (Intel's official CLI, part of the oneAPI Level Zero +// userspace stack). When installed it returns a stable JSON view of +// every Intel GPU on the host — dGPU (Arc, Data Center Max/Flex) or +// iGPU — with a PCI BDF address we key on and, where the driver +// reports it, a total-VRAM figure. +// +// 2. sysfs (/sys/class/drm/card*). Present on every host with an Intel +// GPU regardless of extra software. We enumerate cards whose vendor +// is Intel (0x8086) and pull the marketing name via ghw, plus any +// mem_info_vram_total the DRM driver exposes (Arc dGPUs report it; +// iGPUs share system RAM and do not). +// +// The dynamic counterparts — VRAM used and busy percent — live in +// stats_linux.go under the same two-tier preference: xpu-smi for full +// telemetry, sysfs mem_info_vram_used as a memory-only fallback. When +// neither source can be read the Intel GPU still appears in the +// inventory, just without live stats — mirroring the same "unknown +// means omitted" convention nvidia-smi degradation follows. +// +// Every Intel adapter is stamped with a PCI BDF-derived statsKey +// ("intel-pci-0000:03:00.0") so the stats collector can join dynamic +// samples back to the static GPUInfo the same way nvidia-smi joins by +// UUID. The prefix keeps the key space disjoint from nvidia-smi UUIDs. + +const ( + // xpuSmiTimeout caps how long we wait for one xpu-smi invocation. + // The tool is usually well under 200 ms, but a wedged Level Zero + // driver can hang; a hard ceiling keeps the per-tick stats collector + // from blocking indefinitely. + xpuSmiTimeout = 3 * time.Second + + // intelPCIVendorID is the PCI vendor ID assigned to Intel. Every + // entry under /sys/class/drm/card*/device/vendor reports this exact + // literal (lowercase, "0x" prefix) for an Intel GPU. + intelPCIVendorID = "0x8086" + + // intelStatsKeyPrefix disambiguates Intel adapter keys from nvidia-smi + // UUIDs in the shared stats map. Keep any prefix change in lock-step + // with the stats collector. + intelStatsKeyPrefix = "intel-pci-" +) + +// xpuSmiDiscoveryDevice mirrors the subset of xpu-smi's `discovery --json` +// entries we consume. Only the fields relevant to inventory are pulled; +// extra keys the tool may add in future versions are ignored by +// encoding/json. +// +// memory_physical_size_byte is present on discrete Intel GPUs (Arc, +// Data Center) and absent (or zero) on iGPUs that share system memory. +// It arrives as either a numeric or string on different xpu-smi builds, +// so we parse it defensively in intelMemoryBytes. +type xpuSmiDiscoveryDevice struct { + DeviceID int `json:"device_id"` + DeviceName string `json:"device_name"` + DeviceType string `json:"device_type"` + DeviceFunctionType string `json:"device_function_type"` + UUID string `json:"uuid"` + PCIBDFAddress string `json:"pci_bdf_address"` + PCIDeviceID string `json:"pci_device_id"` + VendorName string `json:"vendor_name"` + MemoryPhysicalSizeByte json.RawMessage `json:"memory_physical_size_byte"` +} + +type xpuSmiDiscoveryEnvelope struct { + DeviceList []xpuSmiDiscoveryDevice `json:"device_list"` +} + +// detectIntelGPUs returns every Intel GPU the host exposes, or nil when +// no Intel adapter can be identified. Prefers xpu-smi, falls back to a +// sysfs walk. A nil return means "no Intel GPU found by any source"; an +// empty slice is never returned. +func detectIntelGPUs() []GPUInfo { + if gpus, ok := detectIntelViaXpuSmi(); ok { + return gpus + } + return detectIntelViaSysfs() +} + +// detectIntelViaXpuSmi runs `xpu-smi discovery --json` and folds the +// device list into GPUInfo. Returns ok=false when the binary is not on +// PATH, the invocation fails or times out, or the payload is empty — +// callers then fall through to the sysfs walk. +// +// Physical adapters only: xpu-smi enumerates SR-IOV virtual functions +// alongside their parent under the same PCI device, and we skip those +// via device_function_type so a single physical Arc doesn't appear +// multiple times in the inventory. +func detectIntelViaXpuSmi() ([]GPUInfo, bool) { + out, err := xpuSmiJSON(context.Background(), "discovery", "--json") + if err != nil { + slog.Debug("xpu-smi discovery unavailable", "err", err) + return nil, false + } + gpus, ok := parseXpuSmiDiscovery(out) + if !ok || len(gpus) == 0 { + return nil, false + } + return gpus, true +} + +// parseXpuSmiDiscovery decodes the JSON envelope. Returns ok=false when +// the payload does not conform to the discovery shape — a version of +// xpu-smi that emits an unrelated JSON document should degrade to the +// sysfs fallback rather than surface as "no Intel GPUs". +func parseXpuSmiDiscovery(out []byte) ([]GPUInfo, bool) { + var env xpuSmiDiscoveryEnvelope + if err := json.Unmarshal(out, &env); err != nil { + slog.Debug("xpu-smi discovery JSON parse failed", "err", err) + return nil, false + } + gpus := make([]GPUInfo, 0, len(env.DeviceList)) + for _, d := range env.DeviceList { + if !isPhysicalIntelGPU(d) { + continue + } + key := intelStatsKeyFromBDF(d.PCIBDFAddress) + if key == "" { + continue + } + name := strings.TrimSpace(d.DeviceName) + if name == "" { + name = "Intel GPU" + } + gpus = append(gpus, GPUInfo{ + Name: name, + VramBytes: intelMemoryBytes(d.MemoryPhysicalSizeByte), + statsKey: key, + }) + } + return gpus, true +} + +// isPhysicalIntelGPU filters the xpu-smi device list to physical GPU +// entries. device_type distinguishes GPU from other Intel accelerators; +// device_function_type separates physical adapters from SR-IOV virtual +// functions ("virtual"). A missing device_function_type is treated as +// physical for compatibility with xpu-smi builds that don't emit it. +func isPhysicalIntelGPU(d xpuSmiDiscoveryDevice) bool { + if !strings.EqualFold(strings.TrimSpace(d.DeviceType), "GPU") && + d.DeviceType != "" { + return false + } + fn := strings.ToLower(strings.TrimSpace(d.DeviceFunctionType)) + switch fn { + case "", "physical": + return true + default: + return false + } +} + +// intelMemoryBytes decodes memory_physical_size_byte. xpu-smi has +// emitted this field as a number ("1699966976") on some builds and as +// a string ("\"1699966976\"") on others; a missing value or zero is +// treated as "unknown" (VramBytes stays 0, which the omitempty tag +// drops from the wire). +func intelMemoryBytes(raw json.RawMessage) uint64 { + trimmed := strings.TrimSpace(string(raw)) + if trimmed == "" || trimmed == "null" { + return 0 + } + trimmed = strings.Trim(trimmed, `"`) + v, err := strconv.ParseUint(trimmed, 10, 64) + if err != nil { + return 0 + } + return v +} + +// intelStatsKeyFromBDF normalizes a PCI BDF address to the statsKey the +// stats collector joins on. xpu-smi and sysfs both spell the address +// lowercase already, but we normalize defensively so a version that +// reports uppercase (or omits the domain prefix) still joins. +func intelStatsKeyFromBDF(bdf string) string { + bdf = strings.ToLower(strings.TrimSpace(bdf)) + if bdf == "" { + return "" + } + // Some xpu-smi builds omit the "0000:" domain segment. + if !strings.Contains(bdf, ":") { + return "" + } + if strings.Count(bdf, ":") == 1 { + bdf = "0000:" + bdf + } + return intelStatsKeyPrefix + bdf +} + +// xpuSmiJSON runs the xpu-smi CLI with the supplied argv and returns +// stdout. A missing binary (not on PATH) surfaces as an exec error, +// which callers treat as "no xpu-smi telemetry available" and degrade +// silently to sysfs. +func xpuSmiJSON(parent context.Context, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(parent, xpuSmiTimeout) + defer cancel() + return exec.CommandContext(ctx, "xpu-smi", args...).Output() +} + +// detectIntelViaSysfs enumerates /sys/class/drm/card* and returns one +// GPUInfo per Intel PCI device. Marketing names come from ghw (which +// uses the shared pcidb), so a card whose device ID is not in the +// bundled DB still appears — just with a generic name. +// +// mem_info_vram_total is present on Arc dGPUs (i915 and xe drivers). +// iGPUs share system RAM and don't report it, so VramBytes stays 0 +// there and the omitempty tag drops the field from JSON, matching the +// existing behavior for unknown VRAM. +func detectIntelViaSysfs() []GPUInfo { + cards, err := filepath.Glob("/sys/class/drm/card*") + if err != nil { + slog.Debug("sysfs drm scan failed", "err", err) + return nil + } + // filepath.Glob returns entries in lexical order already, but be + // explicit so cardN ordering is stable even if the pattern semantics + // ever change. + sort.Strings(cards) + + seen := make(map[string]struct{}) + var gpus []GPUInfo + names := intelDeviceNamesFromGHW() + + for _, card := range cards { + // Skip connector entries like /sys/class/drm/card0-DP-1: only + // the bare cardN symlink has a device/ directory. + base := filepath.Base(card) + if strings.Contains(base, "-") { + continue + } + vendor, err := os.ReadFile(filepath.Join(card, "device", "vendor")) + if err != nil { + continue + } + if strings.TrimSpace(string(vendor)) != intelPCIVendorID { + continue + } + bdf, err := readSysfsBDF(card) + if err != nil || bdf == "" { + continue + } + key := intelStatsKeyPrefix + bdf + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + + name := names[bdf] + if name == "" { + name = "Intel GPU" + } + gpus = append(gpus, GPUInfo{ + Name: name, + VramBytes: readSysfsVRAMTotal(card), + statsKey: key, + }) + } + return gpus +} + +// readSysfsBDF resolves the PCI BDF address a DRM card is attached to. +// The `device` entry under /sys/class/drm/cardN is a symlink into +// /sys/devices/pci*/..., whose final component is the BDF. +func readSysfsBDF(cardPath string) (string, error) { + target, err := os.Readlink(filepath.Join(cardPath, "device")) + if err != nil { + return "", err + } + bdf := strings.ToLower(filepath.Base(target)) + if !looksLikePCIBDF(bdf) { + return "", nil + } + return bdf, nil +} + +// looksLikePCIBDF applies a cheap shape check so we don't accept a +// non-PCI final path segment (e.g. a platform device) as an adapter +// key. Real BDFs look like "0000:03:00.0" — four colons-and-dots +// separated hex fields. Full lexical validation would add no signal +// beyond the sysfs vendor filter that already gated us here. +func looksLikePCIBDF(s string) bool { + if !strings.Contains(s, ":") || !strings.Contains(s, ".") { + return false + } + return strings.Count(s, ":") == 2 +} + +// readSysfsVRAMTotal returns the DRM-reported total VRAM in bytes, or +// zero when the driver doesn't expose it (iGPU) or the read fails. +func readSysfsVRAMTotal(cardPath string) uint64 { + data, err := os.ReadFile(filepath.Join(cardPath, "device", "mem_info_vram_total")) + if err != nil { + return 0 + } + v, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0 + } + return v +} + +// readSysfsVRAMUsed returns the DRM-reported bytes of VRAM in use, or +// (0, false) when the driver doesn't expose it or the read fails. This +// is the sysfs counterpart to xpu-smi's dynamic stats and is consumed +// from stats_linux.go. +func readSysfsVRAMUsed(cardPath string) (uint64, bool) { + data, err := os.ReadFile(filepath.Join(cardPath, "device", "mem_info_vram_used")) + if err != nil { + return 0, false + } + v, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0, false + } + return v, true +} + +// intelDeviceNamesFromGHW returns a map from PCI BDF ("0000:03:00.0") +// to marketing name for every Intel graphics adapter ghw can see. ghw +// already parses the shared pcidb; we reuse that here rather than +// re-shipping the mapping ourselves. A ghw failure returns an empty +// map — callers substitute "Intel GPU" for any adapter whose name +// isn't found. +func intelDeviceNamesFromGHW() map[string]string { + out := map[string]string{} + gpu, err := ghw.GPU() + if err != nil { + slog.Debug("ghw graphics enumeration failed for Intel name lookup", "err", err) + return out + } + for _, card := range gpu.GraphicsCards { + if card.DeviceInfo == nil || card.DeviceInfo.Vendor == nil { + continue + } + // pcidb reports vendor IDs as unprefixed lowercase hex ("8086"); + // sysfs and xpu-smi use "0x8086". Compare against the sysfs + // form so both sides key on the same literal. + vendorID := "0x" + strings.ToLower(strings.TrimSpace(card.DeviceInfo.Vendor.ID)) + if vendorID != intelPCIVendorID { + continue + } + bdf := strings.ToLower(strings.TrimSpace(card.Address)) + if bdf == "" { + continue + } + // ghw sometimes reports the BDF without the "0000:" PCI domain; + // normalize to the same shape sysfs uses so the lookup joins. + if strings.Count(bdf, ":") == 1 { + bdf = "0000:" + bdf + } + if card.DeviceInfo.Product == nil || card.DeviceInfo.Product.Name == "" { + continue + } + out[bdf] = card.DeviceInfo.Product.Name + } + return out +} diff --git a/services/nvpair-node-info/gpu_intel_linux_test.go b/services/nvpair-node-info/gpu_intel_linux_test.go new file mode 100644 index 00000000..78671430 --- /dev/null +++ b/services/nvpair-node-info/gpu_intel_linux_test.go @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "reflect" + "testing" +) + +// TestParseXpuSmiDiscovery pins the JSON schema the Intel discovery +// parser accepts. If this ever drifts, every Intel host would list its +// GPUs without VRAM (or without a statsKey, breaking the dynamic-stats +// join). The cases target: a physical Arc dGPU with a numeric memory +// size, a physical Arc with the string-encoded variant xpu-smi emits +// on some builds, a virtual function that must be skipped, and a +// non-GPU device that must be ignored. +func TestParseXpuSmiDiscovery(t *testing.T) { + const payload = `{ + "device_list": [ + { + "device_id": 0, + "device_type": "GPU", + "device_function_type": "physical", + "device_name": "Intel(R) Arc(TM) A770 Graphics", + "uuid": "00000000-0000-0000-0000-56a05e001000", + "pci_bdf_address": "0000:03:00.0", + "pci_device_id": "0x56A0", + "vendor_name": "Intel(R) Corporation", + "memory_physical_size_byte": "17102532608" + }, + { + "device_id": 1, + "device_type": "GPU", + "device_function_type": "physical", + "device_name": "Intel(R) Arc(TM) A580 Graphics", + "pci_bdf_address": "0000:04:00.0", + "memory_physical_size_byte": 8589934592 + }, + { + "device_id": 2, + "device_type": "GPU", + "device_function_type": "virtual", + "device_name": "Intel(R) Arc VF", + "pci_bdf_address": "0000:03:00.1" + }, + { + "device_id": 3, + "device_type": "NNP", + "device_name": "Intel(R) Habana Gaudi", + "pci_bdf_address": "0000:05:00.0" + } + ] + }` + + got, ok := parseXpuSmiDiscovery([]byte(payload)) + if !ok { + t.Fatal("parseXpuSmiDiscovery returned ok=false on a valid payload") + } + if len(got) != 2 { + t.Fatalf("got %d GPUs, want 2 (VF and non-GPU must be skipped): %+v", len(got), got) + } + + if got[0].Name != "Intel(R) Arc(TM) A770 Graphics" { + t.Errorf("gpu 0 Name = %q, want Arc A770", got[0].Name) + } + if got[0].VramBytes != 17102532608 { + t.Errorf("gpu 0 VramBytes = %d, want 17102532608 (string-encoded)", got[0].VramBytes) + } + if got[0].statsKey != "intel-pci-0000:03:00.0" { + t.Errorf("gpu 0 statsKey = %q, want intel-pci-0000:03:00.0", got[0].statsKey) + } + + if got[1].VramBytes != 8589934592 { + t.Errorf("gpu 1 VramBytes = %d, want 8589934592 (numeric)", got[1].VramBytes) + } + if got[1].statsKey != "intel-pci-0000:04:00.0" { + t.Errorf("gpu 1 statsKey = %q, want intel-pci-0000:04:00.0", got[1].statsKey) + } +} + +// TestParseXpuSmiDiscoveryRejectsGarbage confirms that a non-conforming +// payload degrades cleanly to the sysfs fallback (ok=false) instead of +// producing a phantom adapter. +func TestParseXpuSmiDiscoveryRejectsGarbage(t *testing.T) { + if _, ok := parseXpuSmiDiscovery([]byte("not json")); ok { + t.Fatal("parseXpuSmiDiscovery accepted non-JSON input") + } +} + +// TestIntelStatsKeyFromBDF covers the sysfs and xpu-smi BDF forms plus +// the domain-less short form some xpu-smi builds emit. A missing colon +// is rejected: without a bus:device separator we can't uniquely +// identify a PCI adapter. +func TestIntelStatsKeyFromBDF(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"0000:03:00.0", "intel-pci-0000:03:00.0"}, + {"03:00.0", "intel-pci-0000:03:00.0"}, + {"0000:03:00.1", "intel-pci-0000:03:00.1"}, + {"0000:AB:CD.0", "intel-pci-0000:ab:cd.0"}, + {"", ""}, + {"3-00-0", ""}, + {"garbage", ""}, + } + for _, c := range cases { + got := intelStatsKeyFromBDF(c.in) + if got != c.want { + t.Errorf("intelStatsKeyFromBDF(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// TestIsPhysicalIntelGPU exercises the filter that keeps SR-IOV virtual +// functions and non-GPU Intel accelerators out of the inventory. The +// "empty device_type" case pins the compatibility path for xpu-smi +// builds that don't emit the field at all. +func TestIsPhysicalIntelGPU(t *testing.T) { + cases := []struct { + name string + in xpuSmiDiscoveryDevice + want bool + }{ + {"physical GPU", xpuSmiDiscoveryDevice{DeviceType: "GPU", DeviceFunctionType: "physical"}, true}, + {"physical GPU with mixed case", xpuSmiDiscoveryDevice{DeviceType: "gpu", DeviceFunctionType: "physical"}, true}, + {"missing function type", xpuSmiDiscoveryDevice{DeviceType: "GPU"}, true}, + {"missing device type is treated as GPU", xpuSmiDiscoveryDevice{DeviceFunctionType: "physical"}, true}, + {"virtual function", xpuSmiDiscoveryDevice{DeviceType: "GPU", DeviceFunctionType: "virtual"}, false}, + {"non-GPU accelerator", xpuSmiDiscoveryDevice{DeviceType: "NNP", DeviceFunctionType: "physical"}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := isPhysicalIntelGPU(c.in); got != c.want { + t.Errorf("isPhysicalIntelGPU(%+v) = %v, want %v", c.in, got, c.want) + } + }) + } +} + +// TestLooksLikePCIBDF pins the shape check the sysfs walker uses to +// filter out platform-device final path segments. Real BDFs have +// exactly two colons and at least one dot; anything else must be +// rejected so a non-PCI DRM entry can't be mistaken for an Intel +// adapter. +func TestLooksLikePCIBDF(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"0000:03:00.0", true}, + {"0000:ab:cd.1", true}, + {"03:00.0", false}, + {"0000-03-00-0", false}, + {"platform:soc0", false}, + {"", false}, + } + for _, c := range cases { + if got := looksLikePCIBDF(c.in); got != c.want { + t.Errorf("looksLikePCIBDF(%q) = %v, want %v", c.in, got, c.want) + } + } +} + +// TestParseIntelXpuSmiDump covers the dump-JSON shapes xpu-smi has +// emitted across releases (bare array vs. object-with-data), the +// header-name variance for utilization and memory-used, and the +// device_id/BDF join paths. A failure in any of these would make +// Intel utilization silently drop from the wire while the Intel +// adapters still appear in the inventory. +func TestParseIntelXpuSmiDump(t *testing.T) { + sources := []intelStatsSource{ + {statsKey: "intel-pci-0000:03:00.0", bdf: "0000:03:00.0", deviceID: 0, hasDevID: true}, + {statsKey: "intel-pci-0000:04:00.0", bdf: "0000:04:00.0", deviceID: 1, hasDevID: true}, + } + + cases := []struct { + name string + in string + want map[string]intelDynamicStat + }{ + { + name: "bare array, MiB memory header", + in: `[ + {"deviceId":"0","GPU Utilization (%)":"42.5","GPU Memory Used (MiB)":"1024"}, + {"deviceId":"1","GPU Utilization (%)":"7","GPU Memory Used (MiB)":"128"} + ]`, + want: map[string]intelDynamicStat{ + "intel-pci-0000:03:00.0": {util: 43, mem: 1024 * 1024 * 1024, hasUtil: true, hasMem: true}, + "intel-pci-0000:04:00.0": {util: 7, mem: 128 * 1024 * 1024, hasUtil: true, hasMem: true}, + }, + }, + { + name: "object with data field, byte memory header", + in: `{"data":[ + {"device_id":0,"XPUM_STATS_GPU_UTILIZATION":15,"XPUM_STATS_MEMORY_USED":536870912} + ]}`, + want: map[string]intelDynamicStat{ + "intel-pci-0000:03:00.0": {util: 15, mem: 536870912, hasUtil: true, hasMem: true}, + }, + }, + { + name: "join by BDF when device_id is absent", + in: `[ + {"pci_bdf_address":"0000:04:00.0","GPU Utilization (%)":"50","GPU Memory Used (MiB)":"64"} + ]`, + want: map[string]intelDynamicStat{ + "intel-pci-0000:04:00.0": {util: 50, mem: 64 * 1024 * 1024, hasUtil: true, hasMem: true}, + }, + }, + { + name: "N/A utilization drops the utilization field, keeps memory", + in: `[ + {"deviceId":"0","GPU Utilization (%)":"N/A","GPU Memory Used (MiB)":"32"} + ]`, + want: map[string]intelDynamicStat{ + "intel-pci-0000:03:00.0": {mem: 32 * 1024 * 1024, hasMem: true}, + }, + }, + { + name: "row for unknown adapter is skipped", + in: `[ + {"deviceId":"99","GPU Utilization (%)":"10","GPU Memory Used (MiB)":"1"} + ]`, + want: nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := parseIntelXpuSmiDump([]byte(c.in), sources) + if len(got) == 0 && len(c.want) == 0 { + return + } + if !reflect.DeepEqual(got, c.want) { + t.Errorf("parseIntelXpuSmiDump = %+v, want %+v", got, c.want) + } + }) + } +} + +// TestSampleIntelStatsMergesXpuSmiAndSysfs pins the merge rule: xpu-smi +// wins for utilization and memory-used, and sysfs only fills memory +// when xpu-smi didn't provide it. Verified indirectly via the map +// returned by parseIntelXpuSmiDump because sampleIntelStats depends on +// filesystem state we don't want to mock in a unit test. +func TestSampleIntelStatsHandlesEmptySources(t *testing.T) { + out := map[string]gpuStat{} + if got := sampleIntelStats(nil, out); got != 0 { + t.Errorf("sampleIntelStats(nil, ...) returned %d samples, want 0", got) + } + if len(out) != 0 { + t.Errorf("sampleIntelStats(nil, ...) mutated out: %v", out) + } +} + +// TestParseFloatishPercent pins the type-tolerant number parsing that +// keeps the Intel decoder working across xpu-smi builds that emit +// percentages as strings, floats, or integers. +func TestParseFloatishPercent(t *testing.T) { + cases := []struct { + in any + want uint32 + wantOK bool + }{ + {42.5, 43, true}, + {"42.5", 43, true}, + {100, 100, true}, + {"105", 100, true}, + {-1.0, 0, false}, + {"N/A", 0, false}, + {"", 0, false}, + } + for _, c := range cases { + got, ok := parseFloatishPercent(c.in) + if ok != c.wantOK || got != c.want { + t.Errorf("parseFloatishPercent(%v) = %d,%v; want %d,%v", c.in, got, ok, c.want, c.wantOK) + } + } +} diff --git a/services/nvpair-node-info/gpu_linux.go b/services/nvpair-node-info/gpu_linux.go index fdc2fa97..39b5c24a 100644 --- a/services/nvpair-node-info/gpu_linux.go +++ b/services/nvpair-node-info/gpu_linux.go @@ -22,33 +22,63 @@ import ( // collector from blocking indefinitely. const nvidiaSmiTimeout = 3 * time.Second -// detectGPUs enumerates GPUs on Linux. It prefers nvidia-smi, which yields the -// marketing name, total VRAM, and a stable per-GPU UUID we reuse as the join -// key (statsKey) against the dynamic stats collector's snapshot. When -// nvidia-smi is absent — no NVIDIA driver, or an AMD/Intel-only host — it falls -// back to ghw, which reports adapter names but no VRAM and no join key, so -// those hosts list their GPUs without dynamic VRAM/utilization (matching the -// pre-existing non-Windows behavior). +// detectGPUs enumerates GPUs on Linux. Each vendor-specific detector runs +// independently so a mixed host (e.g. NVIDIA + Intel Arc) reports every +// adapter, not just the first one whose driver stack is installed. // -// On unified-memory architectures (UMA, e.g. Grace-Blackwell / DGX Spark) -// nvidia-smi reports [N/A] for memory.total because the GPU shares system -// DRAM; in that case VramBytes is filled from detectMemoryTotal() instead. +// Preferences: +// +// - NVIDIA: nvidia-smi yields the marketing name, total VRAM, and a stable +// per-GPU UUID reused as the join key (statsKey) against the dynamic stats +// collector's snapshot. On unified-memory architectures (UMA, e.g. +// Grace-Blackwell / DGX Spark) nvidia-smi reports [N/A] for memory.total +// because the GPU shares system DRAM; VramBytes is filled from +// detectMemoryTotal() instead. +// +// - Intel: xpu-smi (Intel oneAPI) yields the marketing name, total VRAM +// (dGPU only), and a per-adapter PCI BDF used as the join key. When +// xpu-smi is not installed the sysfs walk under /sys/class/drm/card* +// still identifies every Intel adapter by vendor ID and pulls VRAM +// from mem_info_vram_total when the driver exposes it. +// +// When neither vendor-specific detector produces anything (no NVIDIA driver, +// no Intel adapter, or a pure AMD host) we fall back to ghw. ghw reports +// adapter names but no VRAM and no join key, so those hosts list their GPUs +// without dynamic VRAM/utilization — matching the pre-existing non-Windows +// behavior for unsupported vendors. func detectGPUs() []GPUInfo { - if out, err := nvidiaSmiCSV("uuid,name,memory.total"); err == nil { - if gpus, uma := parseNvidiaStatic(out); len(gpus) > 0 { - if uma { - if total := detectMemoryTotal(); total > 0 { - for i := range gpus { - if gpus[i].usesSystemMemoryUsage { - gpus[i].VramBytes = total - } - } + var gpus []GPUInfo + gpus = append(gpus, detectNvidiaLinux()...) + gpus = append(gpus, detectIntelGPUs()...) + if len(gpus) == 0 { + return detectGPUsGHW() + } + return gpus +} + +// detectNvidiaLinux is the NVIDIA half of detectGPUs, extracted so the +// vendor-specific detectors compose without early-returning past each other. +// Returns nil when nvidia-smi is absent, times out, or produces an empty +// static row set. +func detectNvidiaLinux() []GPUInfo { + out, err := nvidiaSmiCSV("uuid,name,memory.total") + if err != nil { + return nil + } + gpus, uma := parseNvidiaStatic(out) + if len(gpus) == 0 { + return nil + } + if uma { + if total := detectMemoryTotal(); total > 0 { + for i := range gpus { + if gpus[i].usesSystemMemoryUsage { + gpus[i].VramBytes = total } } - return gpus } } - return detectGPUsGHW() + return gpus } // detectGPUsGHW is the ghw-based fallback, identical in spirit to the diff --git a/services/nvpair-node-info/stats_intel_linux.go b/services/nvpair-node-info/stats_intel_linux.go new file mode 100644 index 00000000..167db71f --- /dev/null +++ b/services/nvpair-node-info/stats_intel_linux.go @@ -0,0 +1,458 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "context" + "encoding/json" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" +) + +// Intel per-tick GPU stats collection on Linux. Runs alongside the +// nvidia-smi pass in stats_linux.go: both write into the same +// map[statsKey]gpuStat, keyed disjointly (Intel entries use the +// intel-pci- prefix so they can't collide with nvidia-smi UUIDs). +// +// Two-tier preference: +// +// 1. xpu-smi dump — one batched invocation per tick that returns a +// compact utilization + memory-used snapshot for every Intel adapter +// the Level Zero stack sees. Parsed defensively because the exact +// JSON schema has drifted across xpu-smi releases. +// +// 2. sysfs (mem_info_vram_used) — best-effort memory-used fallback +// when xpu-smi is missing or produced no row for a given adapter. +// Present on Arc dGPUs; iGPUs share system RAM and do not expose it. +// Utilization is not read from sysfs — Intel doesn't publish a +// stable busy-percent counter under DRM, so the field remains +// "unknown" (zero, dropped by omitempty) on hosts without xpu-smi. +// +// The collector caches the set of Intel adapters at startup so per-tick +// work is bounded. A latch on xpu-smi keeps a missing binary from +// re-warning every second, matching the nvidia-smi treatment. + +// intelStatsSource pairs the join key with the sysfs directory the +// sysfs fallback reads from. deviceID is the xpu-smi enumeration index, +// used to correlate a dump row back to this adapter — some xpu-smi +// builds report device_id as the only stable identifier in the dump +// output, others include the BDF; we accept either. +type intelStatsSource struct { + statsKey string + bdf string + deviceID int + sysfsCard string + hasDevID bool +} + +// intelXpuSmiUnavailable latches on the first xpu-smi failure so we +// don't re-spawn (and re-warn about) a missing binary every tick. +var intelXpuSmiUnavailable atomic.Bool + +// discoverIntelStatsSources enumerates the Intel adapters the stats +// collector will sample every tick. Combines sysfs (authoritative for +// the card path used by the memory-used fallback) with xpu-smi's +// device_id (when available) so the batched dump can be joined back to +// each adapter without another discovery call per tick. +func discoverIntelStatsSources() []intelStatsSource { + bySysfs := collectIntelSysfsCards() + byXpuSmi := collectIntelXpuSmiDevices() + + seen := make(map[string]struct{}) + var out []intelStatsSource + + // Prefer the sysfs walk as the source of truth: every card we can + // read memory-used from ends up in the sample set. Enrich with the + // xpu-smi device_id when its BDF matches, so utilization joins. + for _, s := range bySysfs { + if _, dup := seen[s.statsKey]; dup { + continue + } + if x, ok := byXpuSmi[s.bdf]; ok { + s.deviceID = x.deviceID + s.hasDevID = x.hasDevID + } + seen[s.statsKey] = struct{}{} + out = append(out, s) + } + // xpu-smi may know about adapters sysfs doesn't (e.g. an SR-IOV + // setup where /sys/class/drm ordering doesn't line up). Emit those + // too; they'll have no sysfs card path so the memory-used fallback + // silently skips them, which matches every other "one source only" + // path in this collector. + for _, x := range byXpuSmi { + if _, dup := seen[x.statsKey]; dup { + continue + } + seen[x.statsKey] = struct{}{} + out = append(out, x) + } + return out +} + +// collectIntelSysfsCards walks /sys/class/drm/card* and returns one +// entry per Intel adapter, keyed by the same statsKey static discovery +// stamps in GPUInfo. Errors during scan return an empty slice — the +// stats collector treats "no Intel sources" as "nothing to sample". +func collectIntelSysfsCards() []intelStatsSource { + cards, err := filepath.Glob("/sys/class/drm/card*") + if err != nil { + return nil + } + var out []intelStatsSource + for _, card := range cards { + base := filepath.Base(card) + if strings.Contains(base, "-") { + continue + } + vendor, err := os.ReadFile(filepath.Join(card, "device", "vendor")) + if err != nil { + continue + } + if strings.TrimSpace(string(vendor)) != intelPCIVendorID { + continue + } + bdf, err := readSysfsBDF(card) + if err != nil || bdf == "" { + continue + } + out = append(out, intelStatsSource{ + statsKey: intelStatsKeyPrefix + bdf, + bdf: bdf, + sysfsCard: card, + }) + } + return out +} + +// collectIntelXpuSmiDevices calls xpu-smi discovery once to pull the +// device_id → BDF map. Returns a map keyed by BDF so callers can enrich +// sysfs-discovered adapters in place. On any failure the latch is set +// and the map is empty; the stats collector then falls back to sysfs +// alone. +func collectIntelXpuSmiDevices() map[string]intelStatsSource { + if intelXpuSmiUnavailable.Load() { + return nil + } + out, err := xpuSmiJSON(context.Background(), "discovery", "--json") + if err != nil { + if intelXpuSmiUnavailable.CompareAndSwap(false, true) { + slog.Info("xpu-smi unavailable; Intel GPU utilization will not be reported", + "err", err) + } + return nil + } + var env xpuSmiDiscoveryEnvelope + if err := json.Unmarshal(out, &env); err != nil { + return nil + } + res := make(map[string]intelStatsSource, len(env.DeviceList)) + for _, d := range env.DeviceList { + if !isPhysicalIntelGPU(d) { + continue + } + key := intelStatsKeyFromBDF(d.PCIBDFAddress) + if key == "" { + continue + } + bdf := strings.TrimPrefix(key, intelStatsKeyPrefix) + res[bdf] = intelStatsSource{ + statsKey: key, + bdf: bdf, + deviceID: d.DeviceID, + hasDevID: true, + } + } + return res +} + +// sampleIntelStats performs one Intel telemetry pass and folds the +// results into out (keyed by statsKey). Returns the count of adapters +// for which a fresh utilization sample was collected — the stats +// collector uses that to decide whether to advance GPUSampledAt. +// +// The xpu-smi path runs first (one batched invocation for every +// adapter), then the sysfs fallback fills any adapter whose VRAM-used +// slot is still zero. Neither path is mandatory: on a host with no +// xpu-smi and iGPUs only, the map is left untouched and the Intel +// adapters still appear in the inventory without dynamic stats. +func sampleIntelStats(sources []intelStatsSource, out map[string]gpuStat) int { + if len(sources) == 0 { + return 0 + } + utilizationSamples := 0 + xpuStats := sampleIntelXpuSmi(sources) + + for _, src := range sources { + stat := out[src.statsKey] + if x, ok := xpuStats[src.statsKey]; ok { + if x.hasUtil { + stat.UtilizationPct = x.util + utilizationSamples++ + } + if x.hasMem { + stat.VRAMUsed = x.mem + } + } + if stat.VRAMUsed == 0 && src.sysfsCard != "" { + if used, ok := readSysfsVRAMUsed(src.sysfsCard); ok { + stat.VRAMUsed = used + } + } + // Only publish an entry if we actually have a field to report; + // otherwise leave the map alone so an omitempty consumer can't + // misread "we tried and got nothing" as "zero busy". + if stat.UtilizationPct != 0 || stat.VRAMUsed != 0 { + out[src.statsKey] = stat + } + } + return utilizationSamples +} + +// intelDynamicStat is the parsed per-adapter sample returned by +// sampleIntelXpuSmi. Booleans distinguish "we have this field" from +// "the field is legitimately zero" — a fully-idle utilization reading +// is still a valid sample and must count toward the tick's freshness. +type intelDynamicStat struct { + util uint32 + mem uint64 + hasUtil bool + hasMem bool +} + +// sampleIntelXpuSmi runs one batched xpu-smi dump invocation for every +// adapter in sources and returns a map keyed by statsKey. On any +// failure (binary missing, timeout, unexpected schema) it returns nil +// and the caller falls through to sysfs. +// +// Metric IDs: 0 = GPU utilization %, 5 = GPU memory used (bytes). These +// are the two IDs xpu-smi has kept stable across every 1.x release; the +// dump JSON reports them under human-readable column names that vary +// per build ("GPU Utilization (%)", "XPUM_STATS_GPU_UTILIZATION", etc.), +// so we normalize the header lookup. +func sampleIntelXpuSmi(sources []intelStatsSource) map[string]intelDynamicStat { + if intelXpuSmiUnavailable.Load() { + return nil + } + // -d -1 selects all devices; -n 1 requests exactly one snapshot; -m + // 0,5 picks utilization and memory-used. -j asks for JSON. Ancient + // xpu-smi releases reject -j — an unknown flag surfaces as a + // nonzero exit which the latch below then silences. + out, err := xpuSmiJSON(context.Background(), + "dump", "-d", "-1", "-m", "0,5", "-n", "1", "-j") + if err != nil { + if intelXpuSmiUnavailable.CompareAndSwap(false, true) { + slog.Info("xpu-smi dump unavailable; Intel GPU utilization / memory-used will not be reported", + "err", err) + } + return nil + } + return parseIntelXpuSmiDump(out, sources) +} + +// parseIntelXpuSmiDump decodes an `xpu-smi dump -j` payload. The tool +// emits either a top-level JSON array of row objects or an object with +// a "data" / "device_list" array; we accept both. Each row has a +// device-id field and free-form column keys — we match utilization and +// memory-used by keyword rather than by exact name so a header rename +// in a future xpu-smi build still parses. +func parseIntelXpuSmiDump(out []byte, sources []intelStatsSource) map[string]intelDynamicStat { + rows := extractIntelDumpRows(out) + if len(rows) == 0 { + return nil + } + byDeviceID := make(map[int]string, len(sources)) + byBDF := make(map[string]string, len(sources)) + for _, s := range sources { + if s.hasDevID { + byDeviceID[s.deviceID] = s.statsKey + } + if s.bdf != "" { + byBDF[s.bdf] = s.statsKey + } + } + res := make(map[string]intelDynamicStat, len(rows)) + for _, row := range rows { + key := matchIntelDumpRow(row, byDeviceID, byBDF) + if key == "" { + continue + } + stat := intelDynamicStat{} + for k, v := range row { + lk := strings.ToLower(k) + switch { + case strings.Contains(lk, "util"): + if pct, ok := parseFloatishPercent(v); ok { + stat.util = pct + stat.hasUtil = true + } + case strings.Contains(lk, "mem"): + if bytesUsed, ok := parseIntelMemoryValue(k, v); ok { + stat.mem = bytesUsed + stat.hasMem = true + } + } + } + if stat.hasUtil || stat.hasMem { + res[key] = stat + } + } + return res +} + +// extractIntelDumpRows unpacks the top-level shape variations xpu-smi +// dump has emitted across releases: a bare array of row objects, or an +// object with a rows/data/device_list array of the same. Returns nil +// when neither shape matches. +func extractIntelDumpRows(out []byte) []map[string]any { + trimmed := strings.TrimSpace(string(out)) + if trimmed == "" { + return nil + } + if strings.HasPrefix(trimmed, "[") { + var arr []map[string]any + if err := json.Unmarshal(out, &arr); err == nil { + return arr + } + return nil + } + var env map[string]any + if err := json.Unmarshal(out, &env); err != nil { + return nil + } + for _, key := range []string{"data", "rows", "device_list", "metrics"} { + if v, ok := env[key]; ok { + if arr, ok := v.([]any); ok { + rows := make([]map[string]any, 0, len(arr)) + for _, r := range arr { + if m, ok := r.(map[string]any); ok { + rows = append(rows, m) + } + } + return rows + } + } + } + return nil +} + +// matchIntelDumpRow resolves a dump row back to one of the adapters we +// know about. Prefers device_id (an integer) when the row exposes it, +// then falls back to matching on PCI BDF if the row carries one. A row +// that can't be matched is skipped — a version of xpu-smi that reports +// an adapter we didn't discover during startup is safely ignored. +func matchIntelDumpRow(row map[string]any, byDeviceID map[int]string, byBDF map[string]string) string { + for k, v := range row { + lk := strings.ToLower(k) + if strings.Contains(lk, "deviceid") || lk == "device_id" || lk == "device" { + if id, ok := parseInt(v); ok { + if key, ok := byDeviceID[id]; ok { + return key + } + } + } + if strings.Contains(lk, "bdf") || strings.Contains(lk, "pci") { + if s, ok := v.(string); ok { + bdf := strings.ToLower(strings.TrimSpace(s)) + if strings.Count(bdf, ":") == 1 { + bdf = "0000:" + bdf + } + if key, ok := byBDF[bdf]; ok { + return key + } + } + } + } + return "" +} + +// parseFloatishPercent accepts the integer, float, or string forms +// xpu-smi has emitted for a percent column and returns a 0..100 uint32. +// Values above 100 are clamped, matching the nvidia-smi treatment. +func parseFloatishPercent(v any) (uint32, bool) { + f, ok := parseFloat(v) + if !ok { + return 0, false + } + if f < 0 { + return 0, false + } + if f > 100 { + f = 100 + } + return uint32(f + 0.5), true +} + +// parseIntelMemoryValue interprets the value of a memory column. xpu-smi +// dump reports either raw bytes or MiB depending on the column heading; +// we detect MiB from the header text and convert. Unknown units default +// to bytes. +func parseIntelMemoryValue(header string, v any) (uint64, bool) { + f, ok := parseFloat(v) + if !ok || f < 0 { + return 0, false + } + lh := strings.ToLower(header) + switch { + case strings.Contains(lh, "mib") || strings.Contains(lh, "(mib)"): + return uint64(f * 1024 * 1024), true + case strings.Contains(lh, "gib") || strings.Contains(lh, "(gib)"): + return uint64(f * 1024 * 1024 * 1024), true + default: + return uint64(f), true + } +} + +func parseFloat(v any) (float64, bool) { + switch x := v.(type) { + case float64: + return x, true + case float32: + return float64(x), true + case int: + return float64(x), true + case int64: + return float64(x), true + case json.Number: + if f, err := x.Float64(); err == nil { + return f, true + } + case string: + s := strings.TrimSpace(x) + if s == "" || strings.EqualFold(s, "n/a") { + return 0, false + } + if f, err := strconv.ParseFloat(s, 64); err == nil { + return f, true + } + } + return 0, false +} + +func parseInt(v any) (int, bool) { + switch x := v.(type) { + case int: + return x, true + case int64: + return int(x), true + case float64: + return int(x), true + case json.Number: + if i, err := x.Int64(); err == nil { + return int(i), true + } + case string: + s := strings.TrimSpace(x) + if i, err := strconv.Atoi(s); err == nil { + return i, true + } + } + return 0, false +} diff --git a/services/nvpair-node-info/stats_linux.go b/services/nvpair-node-info/stats_linux.go index 30b6f4df..624c7e7a 100644 --- a/services/nvpair-node-info/stats_linux.go +++ b/services/nvpair-node-info/stats_linux.go @@ -75,6 +75,14 @@ type statsCollector struct { // re-spawn (and re-warn about) a missing binary every tick. nvidiaUnavailable atomic.Bool + // intelSources is the set of Intel GPUs the collector samples every + // tick. Cached at startup to keep per-tick work bounded — the sysfs + // walk and xpu-smi discovery run once, and the tick loop only pays + // for the batched sampling call plus one sysfs read per adapter. + // Empty on hosts with no Intel GPU; the sampler skips the pass + // entirely when this is empty, so a pure NVIDIA host pays nothing. + intelSources []intelStatsSource + stop chan struct{} done chan struct{} stopOnce sync.Once @@ -94,6 +102,14 @@ func startStatsCollector() *statsCollector { // Prime the CPU baseline so the first tick produces a real delta rather // than a spurious reading (with no previous sample, util reports 0). c.prevCPU = readCPUTimes() + // Cache the Intel adapter set once so per-tick sampling doesn't repeat + // the sysfs walk and the xpu-smi discovery call. A host that hot-plugs + // an Intel GPU after startup won't pick it up without a restart — + // mirroring the static-once discovery every other detector uses. + c.intelSources = discoverIntelStatsSources() + if n := len(c.intelSources); n > 0 { + slog.Info("Intel GPU stats sources discovered", "count", n) + } go c.run() return c } @@ -150,27 +166,39 @@ func (c *statsCollector) decodeSnapshot() *statsSnapshot { return snap } -// decodeGPU queries nvidia-smi and folds the per-GPU results into out, keyed -// by UUID. On the first failure it latches nvidiaUnavailable so subsequent -// ticks short-circuit silently. Unified-memory usage remains available through -// the independent /proc/meminfo sample assembled by buildResponse. +// decodeGPU folds per-GPU dynamic stats from every supported vendor into out, +// each entry keyed by its statsKey (nvidia-smi UUID for NVIDIA, PCI-BDF for +// Intel — the two spaces are disjoint by construction). Returns true when at +// least one adapter produced a fresh utilization sample this tick, which the +// caller uses to advance GPUSampledAt. Vendor detectors that latch as +// unavailable (missing binary, wedged driver) short-circuit silently and let +// the other detectors keep sampling. func (c *statsCollector) decodeGPU(out map[string]gpuStat) bool { + nvSamples := c.decodeNvidiaGPU(out) + intelSamples := sampleIntelStats(c.intelSources, out) + return nvSamples+intelSamples > 0 +} + +// decodeNvidiaGPU is the nvidia-smi half of decodeGPU. Kept separate so the +// vendor latches don't leak into the Intel path — a host without nvidia-smi +// installed still needs the Intel sample to run. +func (c *statsCollector) decodeNvidiaGPU(out map[string]gpuStat) int { if c.nvidiaUnavailable.Load() { - return false + return 0 } csv, err := nvidiaSmiCSV("uuid,utilization.gpu,memory.used") if err != nil { if c.nvidiaUnavailable.CompareAndSwap(false, true) { - slog.Warn("nvidia-smi unavailable; GPU utilization / dedicated VRAM-used will not be reported", + slog.Warn("nvidia-smi unavailable; NVIDIA GPU utilization / dedicated VRAM-used will not be reported", "err", err) } - return false + return 0 } parsed, utilizationSamples := parseNvidiaDynamic(csv) for k, v := range parsed { out[k] = v } - return utilizationSamples > 0 + return utilizationSamples } // Snapshot returns the latest published statsSnapshot. Safe for concurrent From cae81855c38eb0354300b039d34d6d7fa3f337d9 Mon Sep 17 00:00:00 2001 From: elyerinfox Date: Mon, 14 Sep 2026 06:04:00 +0000 Subject: [PATCH 2/3] Read Intel Arc VRAM via DRM_IOCTL_I915_QUERY Reports real Arc VRAM totals and dynamic memory-used on stock i915 kernels that don't expose the mem_info_vram_* sysfs attributes. Signed-off-by: elyerinfox --- services/nvpair-node-info/README.md | 2 +- .../nvpair-node-info/gpu_intel_drm_linux.go | 239 ++++++++++++++++++ .../gpu_intel_drm_linux_test.go | 132 ++++++++++ services/nvpair-node-info/gpu_intel_linux.go | 27 +- .../nvpair-node-info/stats_intel_linux.go | 52 ++-- 5 files changed, 432 insertions(+), 20 deletions(-) create mode 100644 services/nvpair-node-info/gpu_intel_drm_linux.go create mode 100644 services/nvpair-node-info/gpu_intel_drm_linux_test.go diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index 020f1113..b6dc2c75 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -94,7 +94,7 @@ The service no longer advertises itself over mDNS. Its parent (the broker) regis - **Windows** (first-class): GPU inventory comes from DXGI (vendor-agnostic — NVIDIA, AMD, Intel Arc, and iGPUs enumerate through the same path, including VRAM). Dynamic CPU / VRAM-used / utilization / memory-used numbers come from a persistent PDH query plus `GlobalMemoryStatusEx`. Intel Arc adapters are handled through this same pipeline with no extra software required. - **Linux** (first-class): Multiple vendor detectors run in parallel and their results are combined, so a mixed host reports every adapter: - **NVIDIA**: inventory, dedicated VRAM, and utilization come from `nvidia-smi`. Unified-memory GPUs (Grace-Blackwell / DGX Spark) use the `/proc/meminfo` system-memory snapshot even when dynamic `nvidia-smi` collection is unavailable. - - **Intel (Arc dGPU and iGPU)**: identity comes from `xpu-smi discovery --json` when installed, otherwise from a sysfs walk of `/sys/class/drm/card*` filtered by PCI vendor `0x8086`. Dedicated VRAM totals come from `xpu-smi` or the DRM `mem_info_vram_total` attribute (Arc dGPU only). Dynamic memory-used comes from `xpu-smi dump` or the DRM `mem_info_vram_used` attribute. Utilization requires `xpu-smi`; without it the field is omitted (Intel does not expose a stable busy-percent under DRM). Adapters are joined across sources by PCI BDF (statsKey prefix `intel-pci-`). + - **Intel (Arc dGPU and iGPU)**: identity comes from `xpu-smi discovery --json` when installed, otherwise from a sysfs walk of `/sys/class/drm/card*` filtered by PCI vendor `0x8086`. Dedicated VRAM total and used bytes come from a direct `DRM_IOCTL_I915_QUERY` memory-regions query against the card's render node (`/dev/dri/renderD*`) — the same source `intel_gpu_top` uses, and the only one that returns real Arc VRAM on stock i915 kernels where the sysfs `mem_info_vram_*` attributes don't exist. If DRM is unreachable, we fall back to `xpu-smi`, then to the DRM `mem_info_vram_total`/`mem_info_vram_used` sysfs attributes (newer i915 / xe builds only). Utilization requires `xpu-smi`; without it the field is omitted (Intel does not expose a stable busy-percent under DRM). Adapters are joined across sources by PCI BDF (statsKey prefix `intel-pci-`). The `render` group must be able to open the render node (default on every Linux distro; PAIR's service supervisor runs as the local user, which is typically in that group). - CPU and system-memory usage come from `/proc`. Adapters from unsupported vendors fall back to names from `ghw` without dynamic GPU stats. - **macOS**: CPU and system-memory usage come from Mach through gopsutil's purego bindings. GPU identity, mapped memory, and utilization come from the built-in, unprivileged `/usr/sbin/ioreg` command's `IOAccelerator` `PerformanceStatistics`; no sudo or private framework binding is required. Apple Silicon is supported directly. Intel/AMD fields are best-effort when their drivers expose the same dedicated-memory counters. The performance keys are undocumented and may change across macOS releases; a missing or changed key leaves only that metric out and does not stop CPU or memory collection. - **Other platforms**: GPU names come from `ghw`; VRAM and dynamic stats are not reported. diff --git a/services/nvpair-node-info/gpu_intel_drm_linux.go b/services/nvpair-node-info/gpu_intel_drm_linux.go new file mode 100644 index 00000000..c2d60c2f --- /dev/null +++ b/services/nvpair-node-info/gpu_intel_drm_linux.go @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "encoding/binary" + "errors" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + "unsafe" +) + +// Direct DRM query for Intel GPU memory regions. The i915 driver in stock +// Debian/Ubuntu kernels does not expose the sysfs mem_info_vram_total / +// mem_info_vram_used attributes that our first-tier sysfs walk relies on — +// those files only appear on very new i915 builds and on the xe driver. On +// every other host with an Arc adapter, /sys/class/drm/card*/device/ has no +// VRAM readout at all, so `intel-gpu-tools` (intel_gpu_top) uses the +// DRM_IOCTL_I915_QUERY ioctl against the render node to get both the +// probed_size (total VRAM) and the unallocated_size (free VRAM) per memory +// region. This file re-implements that query in pure Go so PAIR can report +// real Arc VRAM totals and dynamic VRAM-used without depending on xpu-smi or +// on newer i915 sysfs. +// +// Reference: https://github.com/mkuoppal/intel-gpu-tools/blob/master/tools/intel_gpu_top.c +// (I915_MEMORY_CLASS_DEVICE is the discrete-VRAM class the ioctl reports.) + +const ( + // drmIoctlI915Query = _IOWR('d', 0x40+0x39, struct drm_i915_query). + // struct drm_i915_query is 16 bytes (u32 num_items, u32 flags, u64 + // items_ptr). _IOWR direction is 3, so the encoded value is + // (3<<30) | (16<<16) | ('d'<<8) | 0x79 = 0xC0106479. Stable across + // every Linux release since i915 gained the query interface in 4.14. + drmIoctlI915Query = 0xC0106479 + + // drmI915QueryMemoryRegions selects the memory-regions payload. + // Returned buffer starts with `struct drm_i915_query_memory_regions` + // (u32 num_regions + 3 * u32 padding), followed by one + // `struct drm_i915_memory_region_info` per region. + drmI915QueryMemoryRegions = 4 + + // i915MemoryClassDevice is the discrete-VRAM class returned in + // `region.memory_class`. Class 0 (SYSTEM) is host RAM the driver can + // map for the GPU; class 1 (DEVICE) is what a user thinks of as + // "VRAM" and what shows up on the marketing box. + i915MemoryClassDevice = 1 + + // drmMemoryRegionInfoSize matches the on-wire layout of + // `struct drm_i915_memory_region_info`: + // + // struct drm_i915_gem_memory_class_instance { + // __u16 memory_class; + // __u16 memory_instance; + // } region; // 4 bytes + // __u32 rsvd0; // 4 + // __u64 probed_size; // 8 + // __u64 unallocated_size; // 8 + // __u64 rsvd1[8]; // 64 (reserved padding the kernel emits) + // + // Total: 88 bytes. A regression here would misread every region + // past the first, so keep this literal aligned with the kernel UAPI. + drmMemoryRegionInfoSize = 88 + + // drmMemoryRegionsHeaderSize is the fixed header preceding the + // region array (num_regions u32 + 3 * u32 padding). + drmMemoryRegionsHeaderSize = 16 +) + +// drmI915Query mirrors `struct drm_i915_query` for the ioctl's first +// argument. Kept as a package-private literal so callers pass a +// heap-allocated backing store into the ioctl (ioctl requires stable +// addresses; a Go stack pointer might migrate mid-call). +type drmI915Query struct { + NumItems uint32 + Flags uint32 + ItemsPtr uint64 +} + +// drmI915QueryItem mirrors `struct drm_i915_query_item`. +// A two-pass call fills length on the first pass (data_ptr=0), then +// allocates a buffer of that size and re-issues with data_ptr set. +type drmI915QueryItem struct { + QueryID uint64 + Length int32 + Flags uint32 + DataPtr uint64 +} + +// intelVRAMFromDRM queries a single render node for its discrete VRAM +// totals. Returns (total, used, true) when the ioctl succeeds and a +// class=DEVICE region is present. On any failure it returns ok=false +// and the caller falls back to whatever other source is available. +// +// This is safe to call every stats-collector tick: the DRM query is a +// short kernel-side lookup with no driver serialization, and the +// probed_size stays constant while unallocated_size tracks the live +// residency of driver-owned buffers. +func intelVRAMFromDRM(renderNode string) (total uint64, used uint64, ok bool) { + fd, err := syscall.Open(renderNode, syscall.O_RDONLY|syscall.O_CLOEXEC, 0) + if err != nil { + slog.Debug("open render node failed", "path", renderNode, "err", err) + return 0, 0, false + } + defer syscall.Close(fd) + + // Pass 1: length probe. data_ptr=0 signals "tell me how large the + // payload is". The kernel writes the required length into item.Length + // and returns without touching data_ptr. Any negative length means + // the query isn't supported on this driver (an older i915 build, an + // xe-driven card, etc.). + item := drmI915QueryItem{QueryID: drmI915QueryMemoryRegions} + if err := ioctlI915Query(fd, &item); err != nil { + slog.Debug("i915 query length probe failed", "path", renderNode, "err", err) + return 0, 0, false + } + if item.Length <= 0 || item.Length > 1<<20 { + // A payload above 1 MiB would be well past any conceivable + // region count and almost certainly a driver bug; refuse to + // allocate for it rather than trusting the number. + return 0, 0, false + } + + // Pass 2: fetch the actual bytes into a Go slice. The backing array + // is pinned for the duration of the syscall because Syscall retains + // the pointer until the kernel returns. + buf := make([]byte, item.Length) + item.DataPtr = uint64(uintptr(unsafe.Pointer(&buf[0]))) + if err := ioctlI915Query(fd, &item); err != nil { + slog.Debug("i915 query fetch failed", "path", renderNode, "err", err) + return 0, 0, false + } + return parseI915MemoryRegions(buf) +} + +// ioctlI915Query issues one DRM_IOCTL_I915_QUERY carrying a single +// query item. Split out so the two-pass sequence stays readable and so +// unit tests can inject failures in a future refactor without going +// through the syscall path. +func ioctlI915Query(fd int, item *drmI915QueryItem) error { + q := drmI915Query{ + NumItems: 1, + ItemsPtr: uint64(uintptr(unsafe.Pointer(item))), + } + _, _, errno := syscall.Syscall( + syscall.SYS_IOCTL, + uintptr(fd), + uintptr(drmIoctlI915Query), + uintptr(unsafe.Pointer(&q)), + ) + if errno != 0 { + return errors.New(errno.Error()) + } + return nil +} + +// parseI915MemoryRegions walks the payload the kernel wrote and returns +// the class=DEVICE region's total + used bytes. Rows past the buffer +// are dropped rather than treated as an error; a truncated payload is +// still valid so long as the class=DEVICE row landed in the readable +// prefix. +func parseI915MemoryRegions(buf []byte) (uint64, uint64, bool) { + if len(buf) < drmMemoryRegionsHeaderSize { + return 0, 0, false + } + numRegions := binary.LittleEndian.Uint32(buf[0:4]) + offset := drmMemoryRegionsHeaderSize + for i := uint32(0); i < numRegions; i++ { + if offset+drmMemoryRegionInfoSize > len(buf) { + break + } + memClass := binary.LittleEndian.Uint16(buf[offset : offset+2]) + probed := binary.LittleEndian.Uint64(buf[offset+8 : offset+16]) + unallocated := binary.LittleEndian.Uint64(buf[offset+16 : offset+24]) + offset += drmMemoryRegionInfoSize + if memClass != i915MemoryClassDevice { + continue + } + var used uint64 + if probed >= unallocated { + used = probed - unallocated + } + return probed, used, true + } + return 0, 0, false +} + +// intelRenderNode resolves the /dev/dri/renderD* path a DRM card is +// backed by. Every physical GPU exposes both a primary node (cardN, +// requires GRAPHICS caps) and a render node (renderDNNN, opens with +// only the render group); we prefer the render node because it needs +// no privileged group membership beyond `render` and is the one +// intel_gpu_top uses too. +// +// The mapping lives under /sys/class/drm/cardN/device/drm/renderD*; a +// card that publishes no render node (some virtualized GPUs) returns +// an empty string, which callers treat as "no DRM query available". +func intelRenderNode(cardPath string) string { + matches, err := filepath.Glob(filepath.Join(cardPath, "device", "drm", "renderD*")) + if err != nil || len(matches) == 0 { + return "" + } + sort.Strings(matches) + // The sysfs entry is just the name; the character device lives + // under /dev/dri. + dev := filepath.Join("/dev/dri", filepath.Base(matches[0])) + if _, err := os.Stat(dev); err != nil { + return "" + } + return dev +} + +// intelRenderNodeForBDF is the xpu-smi-path counterpart of +// intelRenderNode: given a PCI BDF (which xpu-smi discovery reports), +// walk /sys/bus/pci/devices//drm/renderD* and return the /dev +// path. Falls back to an empty string when nothing matches, so callers +// treat it as "sysfs is our only vram source for this adapter". +func intelRenderNodeForBDF(bdf string) string { + if bdf == "" { + return "" + } + base := filepath.Join("/sys/bus/pci/devices", strings.ToLower(bdf), "drm") + matches, err := filepath.Glob(filepath.Join(base, "renderD*")) + if err != nil || len(matches) == 0 { + return "" + } + sort.Strings(matches) + dev := filepath.Join("/dev/dri", filepath.Base(matches[0])) + if _, err := os.Stat(dev); err != nil { + return "" + } + return dev +} diff --git a/services/nvpair-node-info/gpu_intel_drm_linux_test.go b/services/nvpair-node-info/gpu_intel_drm_linux_test.go new file mode 100644 index 00000000..ee7eb1e1 --- /dev/null +++ b/services/nvpair-node-info/gpu_intel_drm_linux_test.go @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package main + +import ( + "encoding/binary" + "testing" +) + +// buildRegionsPayload constructs a synthetic drm_i915_query_memory_regions +// buffer with the same on-wire layout the kernel emits. Keeps the parser +// test independent from an actual DRM device. +func buildRegionsPayload(regions []struct { + class uint16 + instance uint16 + probed uint64 + unallocated uint64 +}) []byte { + buf := make([]byte, drmMemoryRegionsHeaderSize+len(regions)*drmMemoryRegionInfoSize) + binary.LittleEndian.PutUint32(buf[0:4], uint32(len(regions))) + // Bytes 4..16 are the header padding the kernel writes as zero. + off := drmMemoryRegionsHeaderSize + for _, r := range regions { + binary.LittleEndian.PutUint16(buf[off:off+2], r.class) + binary.LittleEndian.PutUint16(buf[off+2:off+4], r.instance) + // rsvd0 (u32) stays zero. + binary.LittleEndian.PutUint64(buf[off+8:off+16], r.probed) + binary.LittleEndian.PutUint64(buf[off+16:off+24], r.unallocated) + // rsvd1[8] (64 bytes) stays zero. + off += drmMemoryRegionInfoSize + } + return buf +} + +// TestParseI915MemoryRegionsPicksDeviceClass pins that we surface the +// class=DEVICE region's totals, not the SYSTEM region — the same +// distinction intel_gpu_top makes. A regression here would blast +// system RAM in as VRAM on every Arc host. +func TestParseI915MemoryRegionsPicksDeviceClass(t *testing.T) { + payload := buildRegionsPayload([]struct { + class uint16 + instance uint16 + probed uint64 + unallocated uint64 + }{ + {class: 0, instance: 0, probed: 64 << 30, unallocated: 60 << 30}, // SYSTEM + {class: 1, instance: 0, probed: 6 << 30, unallocated: 5<<30 + 512<<20}, + }) + + total, used, ok := parseI915MemoryRegions(payload) + if !ok { + t.Fatal("parseI915MemoryRegions returned ok=false on a well-formed payload") + } + if total != 6<<30 { + t.Errorf("total = %d, want %d (class=DEVICE probed)", total, 6<<30) + } + // used = probed - unallocated = 6 GiB - (5 GiB + 512 MiB) = 512 MiB. + if want := uint64(512 << 20); used != want { + t.Errorf("used = %d, want %d", used, want) + } +} + +// TestParseI915MemoryRegionsNoDeviceClass verifies the parser reports +// failure when only a SYSTEM region is present. That's how the ioctl +// answers on a machine with no discrete Intel GPU under i915 — we must +// not fabricate a VRAM figure from the SYSTEM row. +func TestParseI915MemoryRegionsNoDeviceClass(t *testing.T) { + payload := buildRegionsPayload([]struct { + class uint16 + instance uint16 + probed uint64 + unallocated uint64 + }{ + {class: 0, instance: 0, probed: 32 << 30, unallocated: 30 << 30}, + }) + if total, used, ok := parseI915MemoryRegions(payload); ok || total != 0 || used != 0 { + t.Fatalf("expected (0,0,false) for system-only payload; got (%d,%d,%v)", total, used, ok) + } +} + +// TestParseI915MemoryRegionsUnallocatedOverProbed guards the arithmetic +// against a driver bug (or a mid-collection race) that reports more +// unallocated than probed. Used should clamp to 0 rather than +// underflowing to a huge number. +func TestParseI915MemoryRegionsUnallocatedOverProbed(t *testing.T) { + payload := buildRegionsPayload([]struct { + class uint16 + instance uint16 + probed uint64 + unallocated uint64 + }{ + {class: 1, instance: 0, probed: 1 << 30, unallocated: 2 << 30}, + }) + total, used, ok := parseI915MemoryRegions(payload) + if !ok || total != 1<<30 || used != 0 { + t.Fatalf("expected clamped used=0; got total=%d used=%d ok=%v", total, used, ok) + } +} + +// TestParseI915MemoryRegionsTruncatedBufferStops confirms we don't +// scan past the end of the buffer when the reported num_regions exceeds +// what fits. Prevents an out-of-bounds slice on a malformed kernel reply. +func TestParseI915MemoryRegionsTruncatedBufferStops(t *testing.T) { + // Header claims 5 regions but only one fits. + buf := make([]byte, drmMemoryRegionsHeaderSize+drmMemoryRegionInfoSize) + binary.LittleEndian.PutUint32(buf[0:4], 5) + // Single region: class=DEVICE, probed=2 GiB, unallocated=1 GiB. + off := drmMemoryRegionsHeaderSize + binary.LittleEndian.PutUint16(buf[off:off+2], i915MemoryClassDevice) + binary.LittleEndian.PutUint64(buf[off+8:off+16], 2<<30) + binary.LittleEndian.PutUint64(buf[off+16:off+24], 1<<30) + + total, used, ok := parseI915MemoryRegions(buf) + if !ok || total != 2<<30 || used != 1<<30 { + t.Fatalf("truncated buffer parse = total=%d used=%d ok=%v; want total=2GiB used=1GiB", total, used, ok) + } +} + +// TestParseI915MemoryRegionsShortBuffer rejects a payload smaller than +// even the header — the ioctl should not produce this, but a defensive +// parser must not panic on it. +func TestParseI915MemoryRegionsShortBuffer(t *testing.T) { + if _, _, ok := parseI915MemoryRegions(nil); ok { + t.Fatal("expected failure on nil buffer") + } + if _, _, ok := parseI915MemoryRegions(make([]byte, 4)); ok { + t.Fatal("expected failure on 4-byte buffer") + } +} diff --git a/services/nvpair-node-info/gpu_intel_linux.go b/services/nvpair-node-info/gpu_intel_linux.go index 531ffee6..b4b0c993 100644 --- a/services/nvpair-node-info/gpu_intel_linux.go +++ b/services/nvpair-node-info/gpu_intel_linux.go @@ -147,9 +147,20 @@ func parseXpuSmiDiscovery(out []byte) ([]GPUInfo, bool) { if name == "" { name = "Intel GPU" } + vram := intelMemoryBytes(d.MemoryPhysicalSizeByte) + if vram == 0 { + // Some xpu-smi builds omit memory_physical_size_byte for + // discrete Arc adapters. Ask the DRM ioctl directly the same + // way the sysfs path does — cheap, reliable, no extra deps. + if node := intelRenderNodeForBDF(strings.TrimPrefix(key, intelStatsKeyPrefix)); node != "" { + if total, _, ok := intelVRAMFromDRM(node); ok { + vram = total + } + } + } gpus = append(gpus, GPUInfo{ Name: name, - VramBytes: intelMemoryBytes(d.MemoryPhysicalSizeByte), + VramBytes: vram, statsKey: key, }) } @@ -274,9 +285,21 @@ func detectIntelViaSysfs() []GPUInfo { if name == "" { name = "Intel GPU" } + vram := readSysfsVRAMTotal(card) + if vram == 0 { + // Stock i915 builds don't expose mem_info_vram_total; fall back + // to the DRM_IOCTL_I915_QUERY memory-regions query the way + // intel_gpu_top does. Reports real Arc VRAM (6/8/16 GiB) even + // when Resizable BAR is off and sysfs shows nothing. + if node := intelRenderNode(card); node != "" { + if total, _, ok := intelVRAMFromDRM(node); ok { + vram = total + } + } + } gpus = append(gpus, GPUInfo{ Name: name, - VramBytes: readSysfsVRAMTotal(card), + VramBytes: vram, statsKey: key, }) } diff --git a/services/nvpair-node-info/stats_intel_linux.go b/services/nvpair-node-info/stats_intel_linux.go index 167db71f..5b1c9ff9 100644 --- a/services/nvpair-node-info/stats_intel_linux.go +++ b/services/nvpair-node-info/stats_intel_linux.go @@ -43,13 +43,17 @@ import ( // sysfs fallback reads from. deviceID is the xpu-smi enumeration index, // used to correlate a dump row back to this adapter — some xpu-smi // builds report device_id as the only stable identifier in the dump -// output, others include the BDF; we accept either. +// output, others include the BDF; we accept either. renderNode is the +// /dev/dri/renderD* path the DRM_IOCTL_I915_QUERY memory-regions fallback +// opens; empty on adapters where no render node was found (very unusual +// but possible under some virtualization). type intelStatsSource struct { - statsKey string - bdf string - deviceID int - sysfsCard string - hasDevID bool + statsKey string + bdf string + deviceID int + sysfsCard string + renderNode string + hasDevID bool } // intelXpuSmiUnavailable latches on the first xpu-smi failure so we @@ -78,6 +82,9 @@ func discoverIntelStatsSources() []intelStatsSource { if x, ok := byXpuSmi[s.bdf]; ok { s.deviceID = x.deviceID s.hasDevID = x.hasDevID + if s.renderNode == "" { + s.renderNode = x.renderNode + } } seen[s.statsKey] = struct{}{} out = append(out, s) @@ -124,9 +131,10 @@ func collectIntelSysfsCards() []intelStatsSource { continue } out = append(out, intelStatsSource{ - statsKey: intelStatsKeyPrefix + bdf, - bdf: bdf, - sysfsCard: card, + statsKey: intelStatsKeyPrefix + bdf, + bdf: bdf, + sysfsCard: card, + renderNode: intelRenderNode(card), }) } return out @@ -164,10 +172,11 @@ func collectIntelXpuSmiDevices() map[string]intelStatsSource { } bdf := strings.TrimPrefix(key, intelStatsKeyPrefix) res[bdf] = intelStatsSource{ - statsKey: key, - bdf: bdf, - deviceID: d.DeviceID, - hasDevID: true, + statsKey: key, + bdf: bdf, + deviceID: d.DeviceID, + renderNode: intelRenderNodeForBDF(bdf), + hasDevID: true, } } return res @@ -179,10 +188,14 @@ func collectIntelXpuSmiDevices() map[string]intelStatsSource { // collector uses that to decide whether to advance GPUSampledAt. // // The xpu-smi path runs first (one batched invocation for every -// adapter), then the sysfs fallback fills any adapter whose VRAM-used -// slot is still zero. Neither path is mandatory: on a host with no -// xpu-smi and iGPUs only, the map is left untouched and the Intel -// adapters still appear in the inventory without dynamic stats. +// adapter), then two fallbacks fill any adapter whose VRAM-used slot +// is still zero: the DRM_IOCTL_I915_QUERY memory-regions ioctl (works +// on every stock i915 kernel — same source intel_gpu_top uses), and +// finally the DRM mem_info_vram_used sysfs attribute (only present on +// newer i915 / xe builds). Neither utilization nor memory-used is +// mandatory: on a host with iGPUs only and no xpu-smi, the map is +// left untouched and the Intel adapters still appear in the inventory +// without dynamic stats. func sampleIntelStats(sources []intelStatsSource, out map[string]gpuStat) int { if len(sources) == 0 { return 0 @@ -201,6 +214,11 @@ func sampleIntelStats(sources []intelStatsSource, out map[string]gpuStat) int { stat.VRAMUsed = x.mem } } + if stat.VRAMUsed == 0 && src.renderNode != "" { + if _, used, ok := intelVRAMFromDRM(src.renderNode); ok { + stat.VRAMUsed = used + } + } if stat.VRAMUsed == 0 && src.sysfsCard != "" { if used, ok := readSysfsVRAMUsed(src.sysfsCard); ok { stat.VRAMUsed = used From f403034de682a053c3332b38e461803a822e6f69 Mon Sep 17 00:00:00 2001 From: elyerinfox Date: Mon, 14 Sep 2026 06:13:25 +0000 Subject: [PATCH 3/3] Bump nvpair-node-info to 0.14.0 MINOR: adds Intel Arc GPU inventory to /v1/node-info on Linux (additive HTTP surface). Rolls the product/installer line to 0.92.0. Signed-off-by: elyerinfox --- services/versions.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/versions.json b/services/versions.json index 29d8c230..c58e3068 100644 --- a/services/versions.json +++ b/services/versions.json @@ -1,11 +1,11 @@ { "$comment": "Single source of truth for all version numbers. See VERSIONING.md for bump rules.", - "product": "0.91.7", - "installer": "0.91.7", + "product": "0.92.0", + "installer": "0.92.0", "components": { "ollama-proxy": "0.26.2", "lmstudio-proxy": "0.16.2", - "nvpair-node-info": "0.13.3", + "nvpair-node-info": "0.14.0", "nvpair-node-scanner": "0.20.3", "nvpair-manual-nodes": "0.11.1", "nvpair-workload-manager": "0.13.3",