diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index d9b5012e3387..2a16bc8e4525 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -113,6 +113,8 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ --mount=type=secret,id=github_clone_token \ . /opt/docker/common/github_auth.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_ucx.sh && \ + bash /opt/docker/common/install_efa_installer.sh && \ + bash /opt/docker/common/install_gdrcopy.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_nixl.sh && \ bash /opt/docker/common/install_etcd.sh && \ rm -rf /root/.cache/uv/archive-v0 && \ diff --git a/docker/common/install_base.sh b/docker/common/install_base.sh index d7df3651dc8e..c35e69c49ce7 100644 --- a/docker/common/install_base.sh +++ b/docker/common/install_base.sh @@ -1,4 +1,6 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 set -ex @@ -73,13 +75,11 @@ cleanup() { init_ubuntu() { apt-get update - # libibverbs-dev is installed but libmlx5.so is missing, reinstall the package - apt remove -y ibverbs-providers libibverbs1 - apt-get --reinstall install -y libibverbs-dev apt-get install -y --no-install-recommends \ libtool \ autoconf \ automake \ + pkg-config \ ccache \ gdb \ git-lfs \ diff --git a/docker/common/install_efa_installer.sh b/docker/common/install_efa_installer.sh new file mode 100755 index 000000000000..b12418302f3b --- /dev/null +++ b/docker/common/install_efa_installer.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -ex + +# EFA Installer 1.51.0 supports --build-ngc only on Ubuntu. +# Other distributions fall back to standard installation, which may change +# system RDMA dependencies. Skip them to preserve the base image's RDMA stack. +. /etc/os-release +if [ "${ID}" != ubuntu ]; then + echo "Skipping EFA installer: OS '${ID}' version '${VERSION_ID}'." + exit 0 +fi + +# NGC PyTorch 26.09 bundles EFA libraries here; the official installer updates +# that existing installation in place. Use the same path for runtime lookup. +EFA_LIB_DIR=/opt/amazon/efa/lib +EFA_INSTALLER_VERSION="${EFA_INSTALLER_VERSION:-1.51.0}" +# TODO: Use an internal mirror once the pinned EFA installer is available there. +EFA_INSTALLER_URL="https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "${WORK_DIR}"' EXIT +cd "${WORK_DIR}" + +curl --connect-timeout 30 --max-time 1800 --retry 3 --retry-delay 2 \ + -fsSL -o "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \ + "${EFA_INSTALLER_URL}" +tar -xf "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" +cd aws-efa-installer + +# Preserve the base image's MPI and rdma-core. +apt-get update +# --build-ngc handles userspace-only installation for Ubuntu containers. +./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc + +# Prefer the EFA runtime over an older distribution libfabric in the loader cache. +printf '%s\n' "${EFA_LIB_DIR}" > /etc/ld.so.conf.d/000_efa.conf +ldconfig diff --git a/docker/common/install_gdrcopy.sh b/docker/common/install_gdrcopy.sh new file mode 100644 index 000000000000..94407fb767c2 --- /dev/null +++ b/docker/common/install_gdrcopy.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +have_headers=false +have_library=false +if printf '#include \n' | cc -E -x c - >/dev/null 2>&1; then + have_headers=true +fi +if ldconfig -p | grep 'libgdrapi\.so\.2 ' >/dev/null; then + have_library=true +fi +if ${have_headers} && ${have_library}; then + echo "GDRCOPY_EXISTING_PRESERVED=1" + exit 0 +fi +if ${have_headers} || ${have_library}; then + echo "Incomplete GDRCopy installation; refusing to overwrite existing files." >&2 + exit 1 +fi + +. /etc/os-release +# Match GDRCopy in the NGC PyTorch 26.09 base images. +GDRCOPY_VERSION=2.5.1 +ARCH=$(uname -m) +# TODO: Add an internal CI mirror after verifying the pinned 2.5.1 DEB/RPM packages. +DOWNLOAD_ROOT="https://developer.download.nvidia.com/compute/redist/gdrcopy/CUDA%2013.0" +WORK_DIR=$(mktemp -d) +trap 'rm -rf "${WORK_DIR}"' EXIT +cd "${WORK_DIR}" + +download() { + curl --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 2 \ + -fsSL "${DOWNLOAD_ROOT}/$1" -o "${1##*/}" + sha256sum "${1##*/}" +} + +case "${ID}:${VERSION_ID}:${ARCH}" in + ubuntu:22.04:x86_64|ubuntu:24.04:x86_64) + package="libgdrapi_${GDRCOPY_VERSION}-1_amd64.Ubuntu${VERSION_ID/./_}.deb" + download "ubuntu${VERSION_ID/./_}/x64/${package}" + dpkg -i "${package}" + ;; + ubuntu:22.04:aarch64|ubuntu:24.04:aarch64) + package="libgdrapi_${GDRCOPY_VERSION}-1_arm64.Ubuntu${VERSION_ID/./_}.deb" + download "ubuntu${VERSION_ID/./_}/aarch64/${package}" + dpkg -i "${package}" + ;; + rocky:8*:x86_64) + runtime="gdrcopy-${GDRCOPY_VERSION}-1.el8.x86_64.rpm" + development="gdrcopy-devel-${GDRCOPY_VERSION}-1.el8.noarch.rpm" + download "rhel8/x64/${runtime}" + download "rhel8/x64/${development}" + rpm -qp --requires "${runtime}" "${development}" + dnf install -y "./${runtime}" "./${development}" + ;; + *) + echo "No GDRCopy ${GDRCOPY_VERSION} package selected for ${ID} ${VERSION_ID} ${ARCH}." >&2 + exit 1 + ;; +esac +ldconfig +printf '#include \n' | cc -E -x c - >/dev/null +ldconfig -p | grep 'libgdrapi\.so\.2 ' +echo "GDRCOPY_USERSPACE_INSTALLED=${GDRCOPY_VERSION}" diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index cabc5919077d..37829bc4a0cc 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -1,72 +1,162 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 set -ex +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Authenticate the github.com clone below; no-op when no token is available. -source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh" +source "${SCRIPT_DIR}/github_auth.sh" -GITHUB_URL="https://github.com" -UCX_INSTALL_PATH="/usr/local/ucx/" -CUDA_PATH="/usr/local/cuda" -NIXL_VERSION="v1.5.0" +# Build-only ABI baseline; runtime libfabric is provided separately. +LIBFABRIC_BUILD_VERSION="v1.21.0" +# Source-build fallback from NIXL v1.5.2's manylinux dependency set. +HWLOC_VERSION=2.12.2 +NIXL_VERSION="v1.5.2" + +GITHUB_URL="${GITHUB_MIRROR:-https://github.com}" NIXL_REPO="https://github.com/ai-dynamo/nixl.git" -OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH - -ARCH_NAME="x86_64-linux-gnu" -GDS_PATH="$CUDA_PATH/targets/x86_64-linux" -if [ "$(uname -m)" != "amd64" ] && [ "$(uname -m)" != "x86_64" ]; then - ARCH_NAME="aarch64-linux-gnu" - GDS_PATH="$CUDA_PATH/targets/sbsa-linux" -fi - -if [ -n "${GITHUB_MIRROR}" ]; then - export PIP_INDEX_URL="https://artifactory.pdx.nvidia.com/artifactory/api/pypi/pypi-remote/simple" -fi -pip3 install meson ninja pybind11 setuptools - -git clone --depth 1 -b ${NIXL_VERSION} ${NIXL_REPO} -cd nixl - -CUDA_SO_PATH=$(find "/usr/local" -name "libcuda.so.1" 2>/dev/null | head -n1) - -if [[ -z "$CUDA_SO_PATH" ]]; then - echo "libcuda.so.1 not found" - exit 1 -fi - -CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH -meson setup builddir \ - -Ducx_path=$UCX_INSTALL_PATH \ - -Dcudapath_lib="$CUDA_PATH/lib64" \ - -Dcudapath_inc="$CUDA_PATH/include" \ - -Dgds_path="$GDS_PATH" \ - -Dinstall_headers=true \ - -Ddisable_plugins=POSIX \ - -Dbuild_tests=false \ - -Dbuild_examples=false \ - --buildtype=release - -cd builddir && ninja install -cd ../.. -rm -rf nixl* # Remove NIXL source tree to save space -export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH - -# Consumers import `nixl`, but the build above installs the backend as -# `nixl_cu13`. Install the dispatching shim with --no-deps: the backend it would -# otherwise pull from PyPI bundles a second UCX, which segfaults alongside the -# one torch already links. -pip3 install --no-deps "nixl==${NIXL_VERSION#v}" - -echo "export LD_LIBRARY_PATH=/opt/nvidia/nvda_nixl/lib/${ARCH_NAME}:/opt/nvidia/nvda_nixl/lib64:\$LD_LIBRARY_PATH" >> "${ENV}" -# ninja installs the bindings outside site-packages, so the shim needs PYTHONPATH. This takes -# precedence over the PyPI backend that requirements-dev.txt installs into the test -# environments, so keep that pin in step with NIXL_VERSION above -- otherwise the wheel is -# installed and silently never used. -# Prepend only once: ${ENV} is sourced again by every nested shell. -cat >> "${ENV}" <<'EOF' -case ":${PYTHONPATH:-}:" in - *":/opt/nvidia/nvda_nixl/lib/python3/dist-packages:"*) ;; - *) export PYTHONPATH="/opt/nvidia/nvda_nixl/lib/python3/dist-packages${PYTHONPATH:+:${PYTHONPATH}}" ;; +CUDA_PATH="/usr/local/cuda" +UCX_INSTALL_PATH="/usr/local/ucx/" +HWLOC_INSTALL_PATH="/usr/local/mpi" + +# Build against the oldest supported libfabric ABI, independently of the runtime EFA installation. +build_libfabric_for_nixl() { + wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "${GITHUB_URL}/ofiwg/libfabric/releases/download/${LIBFABRIC_BUILD_VERSION}/libfabric-${LIBFABRIC_BUILD_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 + tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 + cd "libfabric-${LIBFABRIC_BUILD_VERSION#v}" + ./autogen.sh + ./configure --prefix="${LIBFABRIC_BUILD_PATH}" \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa \ + --with-cuda="$CUDA_PATH" \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen + make -j"${NPROC:-$(nproc)}" + make install + cd ../ + rm -rf "libfabric-${LIBFABRIC_BUILD_VERSION#v}" +} + +prepare_hwloc() { + # NGC PyTorch's default Open MPI comes from HPC-X and is exposed at /usr/local/mpi. + # Reuse the hwloc bundled with that MPI installation for the EFA plugin. + if [[ -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" && + -f "${HWLOC_INSTALL_PATH}/lib/libhwloc.so" && + -f "${HWLOC_INSTALL_PATH}/lib/pkgconfig/hwloc.pc" ]]; then + export PKG_CONFIG_PATH="${HWLOC_INSTALL_PATH}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + fi + # The LIBFABRIC plugin requires hwloc >= 2.3. For the current base images: + # - NGC PyTorch: reuse bundled HPC-X hwloc 2.7.1 without rebuilding. + # - Non-NGC SBSA Ubuntu 24.04: reuse system hwloc 2.10.0 without rebuilding. + # - Rocky 8: system hwloc 2.2.0 is too old, so build the fallback below. + # Missing hwloc development files also trigger this fallback on any image. + if ! pkg-config --atleast-version=2.3 hwloc; then + curl --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 2 \ + -fsSL "https://download.open-mpi.org/release/hwloc/v${HWLOC_VERSION%.*}/hwloc-${HWLOC_VERSION}.tar.gz" \ + -o hwloc.tar.gz + tar -xzf hwloc.tar.gz + cd "hwloc-${HWLOC_VERSION}" + # Match NIXL's manylinux dependency; retain it for runtime as well as compilation. + ./configure --prefix=/usr/local --libdir=/usr/local/lib --disable-nvml + make -j"${NPROC:-$(nproc)}" + make install + cd .. + rm -rf "hwloc-${HWLOC_VERSION}" hwloc.tar.gz + export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + # On Rocky 8, the system hwloc RPM and its /usr/lib64 files remain installed. + # Prefer the new /usr/local/lib/libhwloc.so.15 through the loader cache. + # This changes default hwloc selection for other consumers, including Open MPI. + printf '%s\n' /usr/local/lib > /etc/ld.so.conf.d/000_nixl_hwloc.conf + ldconfig + fi +} + +build_and_install_nixl() { + OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH + + ARCH_NAME="x86_64-linux-gnu" + GDS_PATH="$CUDA_PATH/targets/x86_64-linux" + if [ "$(uname -m)" != "amd64" ] && [ "$(uname -m)" != "x86_64" ]; then + ARCH_NAME="aarch64-linux-gnu" + GDS_PATH="$CUDA_PATH/targets/sbsa-linux" + fi + + if [ -n "${GITHUB_MIRROR}" ]; then + export PIP_INDEX_URL="https://artifactory.pdx.nvidia.com/artifactory/api/pypi/pypi-remote/simple" + fi + pip3 install meson ninja pybind11 setuptools + + git clone --depth 1 -b ${NIXL_VERSION} ${NIXL_REPO} + cd nixl + + CUDA_SO_PATH=$(find "/usr/local" -name "libcuda.so.1" 2>/dev/null | head -n1) + + if [[ -z "$CUDA_SO_PATH" ]]; then + echo "libcuda.so.1 not found" + exit 1 + fi + + CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) + + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH + # This build-only prefix is removed on exit; runtime libfabric comes from the image or host. + meson setup builddir \ + -Ducx_path=$UCX_INSTALL_PATH \ + -Dlibfabric_path="${LIBFABRIC_BUILD_PATH}" \ + -Dcudapath_lib="$CUDA_PATH/lib64" \ + -Dcudapath_inc="$CUDA_PATH/include" \ + -Dgds_path="$GDS_PATH" \ + -Dinstall_headers=true \ + -Ddisable_plugins=POSIX \ + -Dbuild_tests=false \ + -Dbuild_examples=false \ + --buildtype=release + + cd builddir && ninja install + # Meson uses dist-packages on Ubuntu and versioned site-packages on Rocky. + NIXL_PYTHON_PATH=$(meson introspect --installed | python3 -c ' +import json, pathlib, sys +paths = {str(pathlib.Path(p).parent.parent) for p in json.load(sys.stdin).values() + if pathlib.Path(p).parent.name == "nixl_cu13" and pathlib.Path(p).name.startswith("_bindings.")} +assert len(paths) == 1, paths +print(paths.pop()) + ') + cd ../.. + rm -rf nixl # Remove NIXL source tree to save space + export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH + + # Consumers import `nixl`, but the build above installs the backend as + # `nixl_cu13`. Install the dispatching shim with --no-deps: the backend it would + # otherwise pull from PyPI bundles a second UCX, which segfaults alongside the + # one torch already links. + pip3 install --no-deps "nixl==${NIXL_VERSION#v}" + + echo "export LD_LIBRARY_PATH=/opt/nvidia/nvda_nixl/lib/${ARCH_NAME}:/opt/nvidia/nvda_nixl/lib64:\$LD_LIBRARY_PATH" >> "${ENV}" + # The bindings are outside Python's default search path, so the shim needs PYTHONPATH. This takes + # precedence over the PyPI backend that requirements-dev.txt installs into the test + # environments, so keep that pin in step with NIXL_VERSION above -- otherwise the wheel is + # installed and silently never used. + # Prepend only once: ${ENV} is sourced again by every nested shell. + cat >> "${ENV}" < -p -t