From 0db1de42f852aae205b0cca66aab8a6a3ed969d2 Mon Sep 17 00:00:00 2001 From: jingxuz Date: Thu, 15 Jan 2026 16:32:11 +0800 Subject: [PATCH] feat: support multinode for JSON output (rebase of #54 onto v0.10) Support multinode when testing with json format output (`-j`), which fixes these issues: - Missing device info of remote nodes in json output - Unexpected local node d2d bandwidth value due to variable `localDevice` not being updated through MPI - MPI_ABORT invoked in multinode broadcast testcases and also in test with option `-j -p multinode` Rebased #54 onto current main, resolving conflicts with the unified multinode/single-node path and the `--pair`/UUID and per-node warning logic added since v0.8. `recordDevices`' caller now passes `deviceCount` instead of `worldSize`, so the JSON and perf-parser outputs iterate the device count rather than the world size (the source of CUDA_ERROR_INVALID_DEVICE on multinode `--json` runs). Co-authored-by: Shady Chan Signed-off-by: Shady Chan Signed-off-by: jingxuz --- json_output.cpp | 19 +++++++++++++-- nvbandwidth.cpp | 2 +- output.cpp | 52 ++++++++++++++++++------------------------ output.h | 5 +++- perf_parser_output.cpp | 6 +++++ 5 files changed, 50 insertions(+), 34 deletions(-) diff --git a/json_output.cpp b/json_output.cpp index 42e50c7..16f627f 100644 --- a/json_output.cpp +++ b/json_output.cpp @@ -167,7 +167,7 @@ void JsonOutput::recordError(const std::vector &errorParts) { } void JsonOutput::recordWarning(const std::string &warning) { - m_root[NVB_TITLE][NVB_WARNING] = warning; + m_root[NVB_TITLE][NVB_WARNING].append(warning); } void JsonOutput::addVersionInfo() { @@ -191,7 +191,22 @@ void JsonOutput::addConfComputeInfo(bool bounceBufferConfComputeEnabled) { void JsonOutput::recordDevices(int deviceCount) { Json::Value deviceList; - +#ifdef MULTINODE + if (worldSize > 1) { + std::vector hostnameExchange(worldSize * STRING_LENGTH); + std::vector deviceNameExchange(worldSize * STRING_LENGTH, 0); + std::vector localDeviceIdExchange(worldSize, -1); + exchangeDeviceInfo(deviceCount, hostnameExchange, deviceNameExchange, localDeviceIdExchange); + for (int i = 0; i < worldSize; i++) { + char *deviceName = &deviceNameExchange[i * STRING_LENGTH]; + std::stringstream buf; + buf << "Process " << getPaddedProcessId(i) << " (" << &hostnameExchange[i * STRING_LENGTH] << "): device " << localDeviceIdExchange[i] << ": " << deviceName; + deviceList.append(buf.str()); + } + m_root[NVB_TITLE][NVB_DEVICE_LIST] = deviceList; + return; + } +#endif for (int iDev = 0; iDev < deviceCount; iDev++) { std::stringstream buf; buf << iDev << ": " << getDeviceDisplayInfo(iDev) << ": (" << env->getHostname() << ")"; diff --git a/nvbandwidth.cpp b/nvbandwidth.cpp index b2c7846..3b99600 100644 --- a/nvbandwidth.cpp +++ b/nvbandwidth.cpp @@ -573,7 +573,7 @@ int main(int argc, char **argv) { output->addConfComputeInfo(gSettings.bounceBufferConfComputeEnabled); // Print GPU information - output->recordDevices(worldSize); + output->recordDevices(deviceCount); // Early CUDA runtime sanity check - test if we can create contexts and allocate memory // This catches driver/runtime issues before they manifest as confusing errors later diff --git a/output.cpp b/output.cpp index a13d78b..36b09e0 100644 --- a/output.cpp +++ b/output.cpp @@ -107,18 +107,14 @@ std::string getDeviceDisplayInfo(int deviceOrdinal) { } #ifdef MULTINODE -// Exchange and print information about all devices in MPI world -// Through this process each process learns about GPUs of other processes, as well as, -// determines its own GPU index -// Each process is allocated a dedicated GPU. It is advisable to initiate NUM_GPU processes per system, -// with each process autonomously selecting a GPU to utilize. To determine this selection, -// processes exchange their hostnames, and look for duplicates of own hostname among processes with lower value of worldRank. -// localRank is equal to number of processes with the same hostname, but lower worldRank. -static void printGPUsMultinode() { +// Exchange information about all devices in the MPI world so each process learns +// about the GPUs selected by every other process. localRank comes from MultiNodeEnv; +// this function assigns localDevice = localRank % deviceCount (unless a --pair UUID +// already selected it) and allgathers hostnames, device names, and ids. +void exchangeDeviceInfo(int deviceCount, std::vector &hostnameExchange, std::vector &deviceNameExchange, std::vector &localDeviceIdExchange) { // Exchange hostnames char hostname[STRING_LENGTH]; gethostname(hostname, STRING_LENGTH); - std::vector hostnameExchange(worldSize * STRING_LENGTH, 0); MPI_Allgather(hostname, STRING_LENGTH, MPI_BYTE, &hostnameExchange[0], STRING_LENGTH, MPI_BYTE, MPI_COMM_WORLD); if (!hasGpuPair()) { @@ -146,24 +142,30 @@ static void printGPUsMultinode() { ASSERT(localDeviceName.size() < STRING_LENGTH); localDeviceName.resize(STRING_LENGTH); - std::vector deviceNameExchange(worldSize * STRING_LENGTH, 0); MPI_Allgather(&localDeviceName[0], STRING_LENGTH, MPI_BYTE, &deviceNameExchange[0], STRING_LENGTH, MPI_BYTE, MPI_COMM_WORLD); // Exchange device ids - std::vector localDeviceIdExchange(worldSize, -1); MPI_Allgather(&localDevice, 1, MPI_INT, &localDeviceIdExchange[0], 1, MPI_INT, MPI_COMM_WORLD); - - // Print gathered info - for (int i = 0; i < worldSize; i++) { - char *deviceName = &deviceNameExchange[i * STRING_LENGTH]; - OUTPUT << "Process " << getPaddedProcessId(i) << " (" << &hostnameExchange[i * STRING_LENGTH] << "): device " - << localDeviceIdExchange[i] << ": " << deviceName << "\n"; - } - OUTPUT << "\n"; } #endif -static void printGPUs() { +void Output::recordDevices(int deviceCount) { +#ifdef MULTINODE + if (worldSize > 1) { + std::vector hostnameExchange(worldSize * STRING_LENGTH); + std::vector deviceNameExchange(worldSize * STRING_LENGTH, 0); + std::vector localDeviceIdExchange(worldSize, -1); + exchangeDeviceInfo(deviceCount, hostnameExchange, deviceNameExchange, localDeviceIdExchange); + // Print gathered info + for (int i = 0; i < worldSize; i++) { + char *deviceName = &deviceNameExchange[i * STRING_LENGTH]; + OUTPUT << "Process " << getPaddedProcessId(i) << " (" << &hostnameExchange[i * STRING_LENGTH] << "): device " + << localDeviceIdExchange[i] << ": " << deviceName << "\n"; + } + OUTPUT << "\n"; + return; + } +#endif OUTPUT << env->getHostname() << "\n"; for (int iDev = 0; iDev < deviceCount; iDev++) { OUTPUT << "Device " << iDev << ": " << getDeviceDisplayInfo(iDev) << "\n"; @@ -171,16 +173,6 @@ static void printGPUs() { OUTPUT << "\n"; } -void Output::recordDevices(int worldSize) { - if (worldSize > 1) { -#ifdef MULTINODE - printGPUsMultinode(); -#endif - } else { - printGPUs(); - } -} - void Output::addTestcase(const std::string &name, const std::string &status, const std::string &msg) { if (status == NVB_RUNNING) { OUTPUT << status << " " << name << ".\n"; diff --git a/output.h b/output.h index 8b74f08..fb2e89d 100644 --- a/output.h +++ b/output.h @@ -94,7 +94,7 @@ class Output { virtual void printInfo(); - virtual void recordDevices(int worldSize); + virtual void recordDevices(int deviceCount); void listTestcases(const std::vector &testcases); }; @@ -102,5 +102,8 @@ class Output { extern Output *output; std::string getDeviceDisplayInfo(int deviceOrdinal); +#ifdef MULTINODE +void exchangeDeviceInfo(int deviceCount, std::vector &hostnameExchange, std::vector &deviceNameExchange, std::vector &localDeviceIdExchange); +#endif #endif // OUTPUT_H_ diff --git a/perf_parser_output.cpp b/perf_parser_output.cpp index 77024f8..d1c175d 100644 --- a/perf_parser_output.cpp +++ b/perf_parser_output.cpp @@ -92,6 +92,12 @@ void PerfParserOutput::addConfComputeInfo(bool bounceBufferConfComputeEnabled) { void PerfParserOutput::recordDevices(int deviceCount) { if (!shouldOutput) return; +#ifdef MULTINODE + if (worldSize > 1) { + std::cout << "Devices: " << worldSize << "\n"; + return; + } +#endif std::cout << "Devices: " << deviceCount << "\n"; }