[None][fix] Enable NIXL LIBFABRIC and update EFA in container images

docker/Dockerfile.multi

@@ -113,6 +113,8 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \
     --mount=type=secret,id=github_clone_token \
     . /opt/docker/common/github_auth.sh && \
     GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_ucx.sh && \
+    bash /opt/docker/common/install_efa_installer.sh && \
+    bash /opt/docker/common/install_gdrcopy.sh && \
     GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_nixl.sh && \
     bash /opt/docker/common/install_etcd.sh && \
     rm -rf /root/.cache/uv/archive-v0 && \

docker/common/install_base.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 
 set -ex
 
@@ -73,13 +75,11 @@ cleanup() {
 
 init_ubuntu() {
   apt-get update
-  # libibverbs-dev is installed but libmlx5.so is missing, reinstall the package
-  apt remove -y ibverbs-providers libibverbs1
-  apt-get --reinstall install -y libibverbs-dev
   apt-get install -y --no-install-recommends \
     libtool \
     autoconf \
     automake \
+    pkg-config \
     ccache \
     gdb \
     git-lfs \

docker/common/install_efa_installer.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+set -ex
+
+# EFA Installer 1.51.0 supports --build-ngc only on Ubuntu.
+# Other distributions fall back to standard installation, which may change
+# system RDMA dependencies. Skip them to preserve the base image's RDMA stack.
+. /etc/os-release
+if [ "${ID}" != ubuntu ]; then
+    echo "Skipping EFA installer: OS '${ID}' version '${VERSION_ID}'."
+    exit 0
+fi
+
+# NGC PyTorch 26.09 bundles EFA libraries here; the official installer updates
+# that existing installation in place. Use the same path for runtime lookup.
+EFA_LIB_DIR=/opt/amazon/efa/lib
+EFA_INSTALLER_VERSION="${EFA_INSTALLER_VERSION:-1.51.0}"
+# TODO: Use an internal mirror once the pinned EFA installer is available there.
+EFA_INSTALLER_URL="https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz"
+
+WORK_DIR="$(mktemp -d)"
+trap 'rm -rf "${WORK_DIR}"' EXIT
+cd "${WORK_DIR}"
+
+curl --connect-timeout 30 --max-time 1800 --retry 3 --retry-delay 2 \
+    -fsSL -o "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \
+    "${EFA_INSTALLER_URL}"
+tar -xf "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz"
+cd aws-efa-installer
+
+# Preserve the base image's MPI and rdma-core.
+apt-get update
+# --build-ngc handles userspace-only installation for Ubuntu containers.
+./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc
+
+# Prefer the EFA runtime over an older distribution libfabric in the loader cache.
+printf '%s\n' "${EFA_LIB_DIR}" > /etc/ld.so.conf.d/000_efa.conf
+ldconfig

docker/common/install_gdrcopy.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+set -euo pipefail
+
+have_headers=false
+have_library=false
+if printf '#include <gdrapi.h>\n' | cc -E -x c - >/dev/null 2>&1; then
+    have_headers=true
+fi
+if ldconfig -p | grep 'libgdrapi\.so\.2 ' >/dev/null; then
+    have_library=true
+fi
+if ${have_headers} && ${have_library}; then
+    echo "GDRCOPY_EXISTING_PRESERVED=1"
+    exit 0
+fi
+if ${have_headers} || ${have_library}; then
+    echo "Incomplete GDRCopy installation; refusing to overwrite existing files." >&2
+    exit 1
+fi
+
+. /etc/os-release
+# Match GDRCopy in the NGC PyTorch 26.09 base images.
+GDRCOPY_VERSION=2.5.1
+ARCH=$(uname -m)
+# TODO: Add an internal CI mirror after verifying the pinned 2.5.1 DEB/RPM packages.
+DOWNLOAD_ROOT="https://developer.download.nvidia.com/compute/redist/gdrcopy/CUDA%2013.0"
+WORK_DIR=$(mktemp -d)
+trap 'rm -rf "${WORK_DIR}"' EXIT
+cd "${WORK_DIR}"
+
+download() {
+    curl --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 2 \
+        -fsSL "${DOWNLOAD_ROOT}/$1" -o "${1##*/}"
+    sha256sum "${1##*/}"
+}
+
+case "${ID}:${VERSION_ID}:${ARCH}" in
+    ubuntu:22.04:x86_64|ubuntu:24.04:x86_64)
+        package="libgdrapi_${GDRCOPY_VERSION}-1_amd64.Ubuntu${VERSION_ID/./_}.deb"
+        download "ubuntu${VERSION_ID/./_}/x64/${package}"
+        dpkg -i "${package}"
+        ;;
+    ubuntu:22.04:aarch64|ubuntu:24.04:aarch64)
+        package="libgdrapi_${GDRCOPY_VERSION}-1_arm64.Ubuntu${VERSION_ID/./_}.deb"
+        download "ubuntu${VERSION_ID/./_}/aarch64/${package}"
+        dpkg -i "${package}"
+        ;;
+    rocky:8*:x86_64)
+        runtime="gdrcopy-${GDRCOPY_VERSION}-1.el8.x86_64.rpm"
+        development="gdrcopy-devel-${GDRCOPY_VERSION}-1.el8.noarch.rpm"
+        download "rhel8/x64/${runtime}"
+        download "rhel8/x64/${development}"
+        rpm -qp --requires "${runtime}" "${development}"
+        dnf install -y "./${runtime}" "./${development}"
+        ;;
+    *)
+        echo "No GDRCopy ${GDRCOPY_VERSION} package selected for ${ID} ${VERSION_ID} ${ARCH}." >&2
+        exit 1
+        ;;
+esac
+ldconfig
+printf '#include <gdrapi.h>\n' | cc -E -x c - >/dev/null
+ldconfig -p | grep 'libgdrapi\.so\.2 '
+echo "GDRCOPY_USERSPACE_INSTALLED=${GDRCOPY_VERSION}"

docker/common/install_nixl.sh

@@ -1,72 +1,162 @@
 #!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 set -ex
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Authenticate the github.com clone below; no-op when no token is available.
-source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh"
+source "${SCRIPT_DIR}/github_auth.sh"
 
-GITHUB_URL="https://github.com"
-UCX_INSTALL_PATH="/usr/local/ucx/"
-CUDA_PATH="/usr/local/cuda"
-NIXL_VERSION="v1.5.0"
+# Build-only ABI baseline; runtime libfabric is provided separately.
+LIBFABRIC_BUILD_VERSION="v1.21.0"
+# Source-build fallback from NIXL v1.5.2's manylinux dependency set.
+HWLOC_VERSION=2.12.2
+NIXL_VERSION="v1.5.2"
+
+GITHUB_URL="${GITHUB_MIRROR:-https://github.com}"
 NIXL_REPO="https://github.com/ai-dynamo/nixl.git"
-OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH
-
-ARCH_NAME="x86_64-linux-gnu"
-GDS_PATH="$CUDA_PATH/targets/x86_64-linux"
-if [ "$(uname -m)" != "amd64" ] && [ "$(uname -m)" != "x86_64" ]; then
-  ARCH_NAME="aarch64-linux-gnu"
-  GDS_PATH="$CUDA_PATH/targets/sbsa-linux"
-fi
-
-if [ -n "${GITHUB_MIRROR}" ]; then
-  export PIP_INDEX_URL="https://artifactory.pdx.nvidia.com/artifactory/api/pypi/pypi-remote/simple"
-fi
-pip3 install meson ninja pybind11 setuptools
-
-git clone --depth 1 -b ${NIXL_VERSION} ${NIXL_REPO}
-cd nixl
-
-CUDA_SO_PATH=$(find "/usr/local" -name "libcuda.so.1" 2>/dev/null | head -n1)
-
-if [[ -z "$CUDA_SO_PATH" ]]; then
-    echo "libcuda.so.1 not found"
-    exit 1
-fi
-
-CUDA_SO_PATH=$(dirname $CUDA_SO_PATH)
-
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH
-meson setup builddir \
-    -Ducx_path=$UCX_INSTALL_PATH \
-    -Dcudapath_lib="$CUDA_PATH/lib64" \
-    -Dcudapath_inc="$CUDA_PATH/include" \
-    -Dgds_path="$GDS_PATH" \
-    -Dinstall_headers=true \
-    -Ddisable_plugins=POSIX \
-    -Dbuild_tests=false \
-    -Dbuild_examples=false \
-    --buildtype=release
-
-cd builddir && ninja install
-cd ../..
-rm -rf nixl*  # Remove NIXL source tree to save space
-export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH
-
-# Consumers import `nixl`, but the build above installs the backend as
-# `nixl_cu13`. Install the dispatching shim with --no-deps: the backend it would
-# otherwise pull from PyPI bundles a second UCX, which segfaults alongside the
-# one torch already links.
-pip3 install --no-deps "nixl==${NIXL_VERSION#v}"
-
-echo "export LD_LIBRARY_PATH=/opt/nvidia/nvda_nixl/lib/${ARCH_NAME}:/opt/nvidia/nvda_nixl/lib64:\$LD_LIBRARY_PATH" >> "${ENV}"
-# ninja installs the bindings outside site-packages, so the shim needs PYTHONPATH. This takes
-# precedence over the PyPI backend that requirements-dev.txt installs into the test
-# environments, so keep that pin in step with NIXL_VERSION above -- otherwise the wheel is
-# installed and silently never used.
-# Prepend only once: ${ENV} is sourced again by every nested shell.
-cat >> "${ENV}" <<'EOF'
-case ":${PYTHONPATH:-}:" in
-  *":/opt/nvidia/nvda_nixl/lib/python3/dist-packages:"*) ;;
-  *) export PYTHONPATH="/opt/nvidia/nvda_nixl/lib/python3/dist-packages${PYTHONPATH:+:${PYTHONPATH}}" ;;
+CUDA_PATH="/usr/local/cuda"
+UCX_INSTALL_PATH="/usr/local/ucx/"
+HWLOC_INSTALL_PATH="/usr/local/mpi"
+
+# Build against the oldest supported libfabric ABI, independently of the runtime EFA installation.
+build_libfabric_for_nixl() {
+    wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \
+        "${GITHUB_URL}/ofiwg/libfabric/releases/download/${LIBFABRIC_BUILD_VERSION}/libfabric-${LIBFABRIC_BUILD_VERSION#v}.tar.bz2" -O libfabric.tar.bz2
+    tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2
+    cd "libfabric-${LIBFABRIC_BUILD_VERSION#v}"
+    ./autogen.sh
+    ./configure --prefix="${LIBFABRIC_BUILD_PATH}" \
+                --disable-verbs \
+                --disable-psm3 \
+                --disable-opx \
+                --disable-usnic \
+                --disable-rstream \
+                --enable-efa \
+                --with-cuda="$CUDA_PATH" \
+                --enable-cuda-dlopen \
+                --with-gdrcopy \
+                --enable-gdrcopy-dlopen
+    make -j"${NPROC:-$(nproc)}"
+    make install
+    cd ../
+    rm -rf "libfabric-${LIBFABRIC_BUILD_VERSION#v}"
+}
+
+prepare_hwloc() {
+    # NGC PyTorch's default Open MPI comes from HPC-X and is exposed at /usr/local/mpi.
+    # Reuse the hwloc bundled with that MPI installation for the EFA plugin.
+    if [[ -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" &&
+          -f "${HWLOC_INSTALL_PATH}/lib/libhwloc.so" &&
+          -f "${HWLOC_INSTALL_PATH}/lib/pkgconfig/hwloc.pc" ]]; then
+        export PKG_CONFIG_PATH="${HWLOC_INSTALL_PATH}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}"
+    fi
+    # The LIBFABRIC plugin requires hwloc >= 2.3. For the current base images:
+    # - NGC PyTorch: reuse bundled HPC-X hwloc 2.7.1 without rebuilding.
+    # - Non-NGC SBSA Ubuntu 24.04: reuse system hwloc 2.10.0 without rebuilding.
+    # - Rocky 8: system hwloc 2.2.0 is too old, so build the fallback below.
+    # Missing hwloc development files also trigger this fallback on any image.
+    if ! pkg-config --atleast-version=2.3 hwloc; then
+        curl --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 2 \
+            -fsSL "https://download.open-mpi.org/release/hwloc/v${HWLOC_VERSION%.*}/hwloc-${HWLOC_VERSION}.tar.gz" \
+            -o hwloc.tar.gz
+        tar -xzf hwloc.tar.gz
+        cd "hwloc-${HWLOC_VERSION}"
+        # Match NIXL's manylinux dependency; retain it for runtime as well as compilation.
+        ./configure --prefix=/usr/local --libdir=/usr/local/lib --disable-nvml
+        make -j"${NPROC:-$(nproc)}"
+        make install
+        cd ..
+        rm -rf "hwloc-${HWLOC_VERSION}" hwloc.tar.gz
+        export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}"
+        # On Rocky 8, the system hwloc RPM and its /usr/lib64 files remain installed.
+        # Prefer the new /usr/local/lib/libhwloc.so.15 through the loader cache.
+        # This changes default hwloc selection for other consumers, including Open MPI.
+        printf '%s\n' /usr/local/lib > /etc/ld.so.conf.d/000_nixl_hwloc.conf
+        ldconfig
+    fi
+}
+
+build_and_install_nixl() {
+    OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH
+
+    ARCH_NAME="x86_64-linux-gnu"
+    GDS_PATH="$CUDA_PATH/targets/x86_64-linux"
+    if [ "$(uname -m)" != "amd64" ] && [ "$(uname -m)" != "x86_64" ]; then
+        ARCH_NAME="aarch64-linux-gnu"
+        GDS_PATH="$CUDA_PATH/targets/sbsa-linux"
+    fi
+
+    if [ -n "${GITHUB_MIRROR}" ]; then
+        export PIP_INDEX_URL="https://artifactory.pdx.nvidia.com/artifactory/api/pypi/pypi-remote/simple"
+    fi
+    pip3 install meson ninja pybind11 setuptools
+
+    git clone --depth 1 -b ${NIXL_VERSION} ${NIXL_REPO}
+    cd nixl
+
+    CUDA_SO_PATH=$(find "/usr/local" -name "libcuda.so.1" 2>/dev/null | head -n1)
+
+    if [[ -z "$CUDA_SO_PATH" ]]; then
+        echo "libcuda.so.1 not found"
+        exit 1
+    fi
+
+    CUDA_SO_PATH=$(dirname $CUDA_SO_PATH)
+
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH
+    # This build-only prefix is removed on exit; runtime libfabric comes from the image or host.
+    meson setup builddir \
+        -Ducx_path=$UCX_INSTALL_PATH \
+        -Dlibfabric_path="${LIBFABRIC_BUILD_PATH}" \
+        -Dcudapath_lib="$CUDA_PATH/lib64" \
+        -Dcudapath_inc="$CUDA_PATH/include" \
+        -Dgds_path="$GDS_PATH" \
+        -Dinstall_headers=true \
+        -Ddisable_plugins=POSIX \
+        -Dbuild_tests=false \
+        -Dbuild_examples=false \
+        --buildtype=release
+
+    cd builddir && ninja install
+    # Meson uses dist-packages on Ubuntu and versioned site-packages on Rocky.
+    NIXL_PYTHON_PATH=$(meson introspect --installed | python3 -c '
+import json, pathlib, sys
+paths = {str(pathlib.Path(p).parent.parent) for p in json.load(sys.stdin).values()
+         if pathlib.Path(p).parent.name == "nixl_cu13" and pathlib.Path(p).name.startswith("_bindings.")}
+assert len(paths) == 1, paths
+print(paths.pop())
+    ')
+    cd ../..
+    rm -rf nixl  # Remove NIXL source tree to save space
+    export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH
+
+    # Consumers import `nixl`, but the build above installs the backend as
+    # `nixl_cu13`. Install the dispatching shim with --no-deps: the backend it would
+    # otherwise pull from PyPI bundles a second UCX, which segfaults alongside the
+    # one torch already links.
+    pip3 install --no-deps "nixl==${NIXL_VERSION#v}"
+
+    echo "export LD_LIBRARY_PATH=/opt/nvidia/nvda_nixl/lib/${ARCH_NAME}:/opt/nvidia/nvda_nixl/lib64:\$LD_LIBRARY_PATH" >> "${ENV}"
+    # The bindings are outside Python's default search path, so the shim needs PYTHONPATH. This takes
+    # precedence over the PyPI backend that requirements-dev.txt installs into the test
+    # environments, so keep that pin in step with NIXL_VERSION above -- otherwise the wheel is
+    # installed and silently never used.
+    # Prepend only once: ${ENV} is sourced again by every nested shell.
+    cat >> "${ENV}" <<EOF
+case ":\${PYTHONPATH:-}:" in
+  *":${NIXL_PYTHON_PATH}:"*) ;;
+  *) export PYTHONPATH="${NIXL_PYTHON_PATH}\${PYTHONPATH:+:\${PYTHONPATH}}" ;;
 esac
 EOF
+}
+
+# Keep build-only libfabric until NIXL is installed; hwloc is retained outside this workspace.
+WORK_DIR=$(mktemp -d)
+trap 'rm -rf "${WORK_DIR}"' EXIT
+LIBFABRIC_BUILD_PATH="${WORK_DIR}/libfabric"
+cd "${WORK_DIR}"
+
+build_libfabric_for_nixl
+prepare_hwloc
+build_and_install_nixl