From bdcdcc814d9c8433ac683c5081bba99e3e37224e Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Thu, 1 Oct 2026 23:15:35 +0000 Subject: [PATCH 01/27] Build NIXL with LIBFABRIC plugin Signed-off-by: Erez Zarum Signed-off-by: Yanchao Lu --- docker/common/install_base.sh | 4 +--- docker/common/install_nixl.sh | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/common/install_base.sh b/docker/common/install_base.sh index a4e6111dec59..c9326c597b6d 100644 --- a/docker/common/install_base.sh +++ b/docker/common/install_base.sh @@ -73,9 +73,6 @@ cleanup() { init_ubuntu() { apt-get update - # libibverbs-dev is installed but libmlx5.so is missing, reinstall the package - apt remove -y ibverbs-providers libibverbs1 - apt-get --reinstall install -y libibverbs-dev apt-get install -y --no-install-recommends \ libtool \ autoconf \ @@ -91,6 +88,7 @@ init_ubuntu() { libstdc++-14-dev \ libnuma1 \ libnuma-dev \ + libhwloc-dev \ python3-dev \ python3-pip \ python-is-python3 \ diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index a27a4c22600c..1ac024e48b3b 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -6,6 +6,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh" GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" +LIBFABRIC_INSTALL_PATH="/opt/amazon/efa" CUDA_PATH="/usr/local/cuda" NIXL_VERSION="v1.4.0" NIXL_REPO="https://github.com/ai-dynamo/nixl.git" @@ -38,6 +39,7 @@ CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH meson setup builddir \ -Ducx_path=$UCX_INSTALL_PATH \ + -Dlibfabric_path=$LIBFABRIC_INSTALL_PATH \ -Dcudapath_lib="$CUDA_PATH/lib64" \ -Dcudapath_inc="$CUDA_PATH/include" \ -Dgds_path="$GDS_PATH" \ From 04069f57655520d538457fdb655ac2acdad0fb97 Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Thu, 1 Oct 2026 23:33:33 +0000 Subject: [PATCH 02/27] disagg - update README to include libfabric changes Signed-off-by: Erez Zarum Signed-off-by: Yanchao Lu --- examples/disaggregated/README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index a03b875a0a6b..1f890de628e6 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -42,7 +42,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX ### LIBFABRIC Backend Setup -**Important Note:** The TensorRT LLM container does not include libfabric or the NIXL-LIBFABRIC plugin by default. You must either rebuild NIXL with libfabric support or provide a pre-compiled plugin. +**Note:** The TensorRT LLM container includes libfabric (via the AWS EFA installation already present in the base image, at `/opt/amazon/efa`) and builds the NIXL-LIBFABRIC plugin against it by default (see `docker/common/install_nixl.sh`). On EFA-enabled instances, no extra installation is needed — just set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. The manual rebuild/pre-compiled-plugin options below are only needed for a custom libfabric installation (e.g. non-EFA fabrics) or a libfabric version other than the one bundled in the container. #### Prerequisites @@ -51,11 +51,13 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX **Required Dependencies:** **Libfabric** -- Custom libfabric installation is available via [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) +- Bundled in the container via the AWS EFA installation at `/opt/amazon/efa`. No action needed on EFA-enabled instances. +- For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) - **Minimum required version:** v1.21.0 -- For EFA-enabled AWS instances, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) +- For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) **hwloc** +- Bundled in the container (`libhwloc-dev`, installed in `docker/common/install_base.sh`). No action needed. - hwloc is used to understand the underlying architecture to optimize application performance - **Suggested version:** 2.10.0 or newer @@ -66,24 +68,26 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX UCX is typically pre-installed in NVIDIA GPU containers. No additional installation is usually required. -#### Installation Options +#### Installation Options (custom libfabric only) -##### Option 1: Rebuild NIXL with LIBFABRIC Support (Recommended) +The container already builds NIXL with the LIBFABRIC plugin against the bundled AWS EFA libfabric. Use these options only if you need a different libfabric installation. + +##### Option 1: Rebuild NIXL with a Custom LIBFABRIC Installation 1. **Install libfabric dependencies:** - Follow the installation instructions from the links above based on your system 2. **Install hwloc:** - - Use your package manager or build from source + - Already bundled in the container (`libhwloc-dev`); only needed if building outside the container 3. **Reinstall NIXL after installing libfabric:** - - After installing libfabric and hwloc, you must rebuild NIXL to generate the LIBFABRIC plugin - - You can base your installation on the TensorRT LLM NIXL installation script located at `docker/common/install_nixl.sh` - - Modify the meson setup command in the script to include the libfabric path: + - After installing libfabric and hwloc, you must rebuild NIXL to generate the LIBFABRIC plugin against your custom path + - You can base your installation on the TensorRT LLM NIXL installation script located at `docker/common/install_nixl.sh`, which already sets `-Dlibfabric_path` for the bundled AWS EFA install + - Point the meson setup command in the script at your custom libfabric path instead: ```bash meson setup builddir \ ... - -Dlibfabric_path=/path/to/libfabric \ # Add this line + -Dlibfabric_path=/path/to/libfabric \ # Override the bundled AWS EFA path --buildtype=release ``` - For more details, see the [NIXL LIBFABRIC Plugin documentation](https://github.com/ai-dynamo/nixl/tree/6ee64753605b3110f8ef96c7cfc2f1315675c9c7/src/plugins/libfabric#nixl-libfabric-plugin) From 4daf6dfdf1c510723ccf1feb8f6e03f0a86d8b91 Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Thu, 1 Oct 2026 23:46:02 +0000 Subject: [PATCH 03/27] disagg - update README troubleshooting section Signed-off-by: Erez Zarum Signed-off-by: Yanchao Lu --- examples/disaggregated/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index 1f890de628e6..1280735c5b64 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -612,7 +612,8 @@ When removing servers, special attention is required in the current version. You **Q: Why does NIXL fail to use LIBFABRIC backend even when `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` is set?** -A: The TensorRT-LLM container doesn't include the NIXL LIBFABRIC plugin by default. You need to either: +A: The container builds the NIXL LIBFABRIC plugin against the bundled AWS EFA libfabric install by default, so this usually means something other than a missing plugin. Check: -1. **Rebuild NIXL**: Install libfabric and hwloc first, then rebuild NIXL following the installation instructions above -2. **Use a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGINS_DIR` to point to its directory +1. **Not running on an EFA-enabled instance**: Confirm EFA devices are present and passed through to the container. +2. **Custom libfabric version mismatch**: If you rebuilt NIXL against a different libfabric installation, confirm it meets the minimum required version (v1.21.0) and that the plugin was built against that same libfabric. +3. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGINS_DIR` to point to its directory. From 4ae328d8db785807f224af9698c47e6a45b3b48b Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Tue, 13 Oct 2026 14:46:27 +0800 Subject: [PATCH 04/27] Fix NIXL builds when bundled EFA is unavailable Only pass /opt/amazon/efa to Meson when both rdma/fabric.h and libfabric.so are present. Otherwise retain system libfabric detection. Qualify the LIBFABRIC setup documentation and share the rebuild instructions with Kubernetes deployments. Cover complete, partial and missing EFA installations with CPU-only script tests. Signed-off-by: Yanchao Lu --- docker/common/install_nixl.sh | 7 ++- examples/disaggregated/README.md | 48 ++++++---------- tests/unittest/scripts/test_install_nixl.py | 63 +++++++++++++++++++++ 3 files changed, 86 insertions(+), 32 deletions(-) create mode 100644 tests/unittest/scripts/test_install_nixl.py diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index 1ac024e48b3b..a424421d4b64 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -7,6 +7,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh" GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" LIBFABRIC_INSTALL_PATH="/opt/amazon/efa" +# Use bundled EFA only when its development files are available. +if [[ ! -f "${LIBFABRIC_INSTALL_PATH}/include/rdma/fabric.h" || + ! -f "${LIBFABRIC_INSTALL_PATH}/lib/libfabric.so" ]]; then + LIBFABRIC_INSTALL_PATH="" +fi CUDA_PATH="/usr/local/cuda" NIXL_VERSION="v1.4.0" NIXL_REPO="https://github.com/ai-dynamo/nixl.git" @@ -39,7 +44,7 @@ CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH meson setup builddir \ -Ducx_path=$UCX_INSTALL_PATH \ - -Dlibfabric_path=$LIBFABRIC_INSTALL_PATH \ + -Dlibfabric_path="${LIBFABRIC_INSTALL_PATH}" \ -Dcudapath_lib="$CUDA_PATH/lib64" \ -Dcudapath_inc="$CUDA_PATH/include" \ -Dgds_path="$GDS_PATH" \ diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index 1280735c5b64..ebab4c1883d5 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -42,7 +42,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX ### LIBFABRIC Backend Setup -**Note:** The TensorRT LLM container includes libfabric (via the AWS EFA installation already present in the base image, at `/opt/amazon/efa`) and builds the NIXL-LIBFABRIC plugin against it by default (see `docker/common/install_nixl.sh`). On EFA-enabled instances, no extra installation is needed — just set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. The manual rebuild/pre-compiled-plugin options below are only needed for a custom libfabric installation (e.g. non-EFA fabrics) or a libfabric version other than the one bundled in the container. +**Note:** Containers built from DLFW base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). With these containers on EFA-enabled instances, no extra library installation is needed — set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. #### Prerequisites @@ -51,13 +51,13 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX **Required Dependencies:** **Libfabric** -- Bundled in the container via the AWS EFA installation at `/opt/amazon/efa`. No action needed on EFA-enabled instances. +- Bundled in DLFW base images with AWS EFA at `/opt/amazon/efa`. No extra library installation is needed when using these containers on EFA-enabled instances. - For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) - **Minimum required version:** v1.21.0 - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) **hwloc** -- Bundled in the container (`libhwloc-dev`, installed in `docker/common/install_base.sh`). No action needed. +- Ubuntu container builds install `libhwloc-dev` in `docker/common/install_base.sh`. For other base images, ensure the hwloc development files are installed before building the plugin. - hwloc is used to understand the underlying architecture to optimize application performance - **Suggested version:** 2.10.0 or newer @@ -68,9 +68,9 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX UCX is typically pre-installed in NVIDIA GPU containers. No additional installation is usually required. -#### Installation Options (custom libfabric only) +#### Installation Options (missing plugin or custom libfabric) -The container already builds NIXL with the LIBFABRIC plugin against the bundled AWS EFA libfabric. Use these options only if you need a different libfabric installation. +Skip these steps if your container already includes the NIXL LIBFABRIC plugin for the libfabric installation you intend to use. ##### Option 1: Rebuild NIXL with a Custom LIBFABRIC Installation @@ -78,18 +78,12 @@ The container already builds NIXL with the LIBFABRIC plugin against the bundled - Follow the installation instructions from the links above based on your system 2. **Install hwloc:** - - Already bundled in the container (`libhwloc-dev`); only needed if building outside the container + - Ubuntu container builds already include `libhwloc-dev`; install the equivalent development package if it is missing from your image 3. **Reinstall NIXL after installing libfabric:** - After installing libfabric and hwloc, you must rebuild NIXL to generate the LIBFABRIC plugin against your custom path - - You can base your installation on the TensorRT LLM NIXL installation script located at `docker/common/install_nixl.sh`, which already sets `-Dlibfabric_path` for the bundled AWS EFA install - - Point the meson setup command in the script at your custom libfabric path instead: - ```bash - meson setup builddir \ - ... - -Dlibfabric_path=/path/to/libfabric \ # Override the bundled AWS EFA path - --buildtype=release - ``` + - You can base your installation on the TensorRT LLM NIXL installation script located at `docker/common/install_nixl.sh`, which selects the bundled AWS EFA path when its header and link library are present, or leaves `-Dlibfabric_path` empty for system detection + - For a custom installation, set `LIBFABRIC_INSTALL_PATH=/path/to/libfabric` immediately before the script's `meson setup` command, keeping the other Meson options unchanged - For more details, see the [NIXL LIBFABRIC Plugin documentation](https://github.com/ai-dynamo/nixl/tree/6ee64753605b3110f8ef96c7cfc2f1315675c9c7/src/plugins/libfabric#nixl-libfabric-plugin) ##### Option 2: Use Pre-compiled LIBFABRIC Plugin @@ -397,9 +391,9 @@ resources: vpc.amazonaws.com/efa: 4 # Number of EFA network interfaces ``` -##### 2. Install EFA Libraries in Container +##### 2. Install EFA Libraries if Missing -AWS EFA library must be installed in the container for LIBFABRIC to work: +Skip this step if the container already includes AWS EFA at `/opt/amazon/efa`. Otherwise, install the EFA libraries before rebuilding NIXL: ```bash # Install AWS EFA library (required for LIBFABRIC with EFA) @@ -408,18 +402,9 @@ tar -xf aws-efa-installer-latest.tar.gz cd aws-efa-installer && ./efa_installer.sh --yes --skip-kmod ``` -##### 3. Rebuild NIXL with EFA Support +##### 3. Rebuild NIXL with EFA Support if Needed -Follow the NIXL rebuild instructions from the LIBFABRIC Backend Setup section, ensuring the libfabric path points to the EFA installation: - -```bash -meson setup builddir \ - -Ducx_path=/usr/local/ucx \ - -Dlibfabric_path=/opt/amazon/efa \ # EFA libfabric installation path - -Dcudapath_lib=/usr/local/cuda/lib64 \ - -Dcudapath_inc=/usr/local/cuda/include \ - --buildtype=release -``` +Skip this step if the NIXL LIBFABRIC plugin was already built against the container's EFA installation. If you installed or replaced EFA, follow the rebuild instructions in [LIBFABRIC Backend Setup](#libfabric-backend-setup), using `/opt/amazon/efa` as the libfabric path. ##### 4. Configure and Launch Services @@ -612,8 +597,9 @@ When removing servers, special attention is required in the current version. You **Q: Why does NIXL fail to use LIBFABRIC backend even when `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` is set?** -A: The container builds the NIXL LIBFABRIC plugin against the bundled AWS EFA libfabric install by default, so this usually means something other than a missing plugin. Check: +A: Check that the container includes the NIXL LIBFABRIC plugin and the dependencies for your fabric: -1. **Not running on an EFA-enabled instance**: Confirm EFA devices are present and passed through to the container. -2. **Custom libfabric version mismatch**: If you rebuilt NIXL against a different libfabric installation, confirm it meets the minimum required version (v1.21.0) and that the plugin was built against that same libfabric. -3. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGINS_DIR` to point to its directory. +1. **Missing plugin**: Older or custom images without libfabric development files at build time may not include the plugin. Follow [LIBFABRIC Backend Setup](#libfabric-backend-setup) to install it. +2. **EFA devices unavailable**: If using EFA, confirm the instance supports it and the devices are passed through to the container. +3. **Custom libfabric version mismatch**: If you rebuilt NIXL against a different libfabric installation, confirm it meets the minimum required version (v1.21.0) and that the plugin was built against that same libfabric. +4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGINS_DIR` to point to its directory. diff --git a/tests/unittest/scripts/test_install_nixl.py b/tests/unittest/scripts/test_install_nixl.py new file mode 100644 index 000000000000..a127d96b4e86 --- /dev/null +++ b/tests/unittest/scripts/test_install_nixl.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.cpu_only + +INSTALL_NIXL = Path(__file__).resolve().parents[3] / "docker/common/install_nixl.sh" + + +@pytest.mark.parametrize("with_header", (False, True)) +@pytest.mark.parametrize("with_library", (False, True)) +def test_libfabric_path_requires_efa_development_files( + tmp_path: Path, with_header: bool, with_library: bool +) -> None: + efa_path = tmp_path / "efa" + for present, name in ( + (with_header, "include/rdma/fabric.h"), + (with_library, "lib/libfabric.so"), + ): + if present: + dependency = efa_path / name + dependency.parent.mkdir(parents=True, exist_ok=True) + dependency.touch() + + # Relocate the installation prefix without needing /opt write access. + installer = tmp_path / "install_nixl.sh" + installer.write_text(INSTALL_NIXL.read_text().replace("/opt/amazon/efa", str(efa_path))) + (tmp_path / "github_auth.sh").write_text(":\n") + meson_args = tmp_path / "meson_args" + command = r""" +pip3() { :; } +git() { mkdir nixl; } +find() { printf '%s\n' /usr/local/cuda/lib64/libcuda.so.1; } +meson() { + printf '%s\n' "$@" > "$MESON_ARGS" + exit 0 +} +source "$INSTALL_NIXL" +""" + result = subprocess.run( + ["bash", "-c", command], + cwd=tmp_path, + env={ + "PATH": os.environ["PATH"], + "INSTALL_NIXL": str(installer), + "MESON_ARGS": str(meson_args), + }, + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + args = meson_args.read_text().splitlines() + expected_path = str(efa_path) if with_header and with_library else "" + assert [arg for arg in args if arg.startswith("-Dlibfabric_path=")] == [ + f"-Dlibfabric_path={expected_path}" + ] From 1179dffd43b1002c63880b9fe964bfc4c480a16c Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Tue, 13 Oct 2026 15:04:13 +0800 Subject: [PATCH 05/27] Clarify the NGC PyTorch EFA installation prefix Document the origin of /opt/amazon/efa in the installer and its regression test. Use the public NGC PyTorch image name in the LIBFABRIC setup guide. Signed-off-by: Yanchao Lu --- docker/common/install_nixl.sh | 2 +- examples/disaggregated/README.md | 4 ++-- tests/unittest/scripts/test_install_nixl.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index a424421d4b64..3d816fba02b6 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -6,8 +6,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh" GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" +# Default prefix for AWS EFA bundled with NGC PyTorch images. LIBFABRIC_INSTALL_PATH="/opt/amazon/efa" -# Use bundled EFA only when its development files are available. if [[ ! -f "${LIBFABRIC_INSTALL_PATH}/include/rdma/fabric.h" || ! -f "${LIBFABRIC_INSTALL_PATH}/lib/libfabric.so" ]]; then LIBFABRIC_INSTALL_PATH="" diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index ebab4c1883d5..f25eadfcd89b 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -42,7 +42,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX ### LIBFABRIC Backend Setup -**Note:** Containers built from DLFW base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). With these containers on EFA-enabled instances, no extra library installation is needed — set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. +**Note:** Containers built from NGC PyTorch base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). With these containers on EFA-enabled instances, no extra library installation is needed — set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. #### Prerequisites @@ -51,7 +51,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX **Required Dependencies:** **Libfabric** -- Bundled in DLFW base images with AWS EFA at `/opt/amazon/efa`. No extra library installation is needed when using these containers on EFA-enabled instances. +- Bundled in NGC PyTorch base images with AWS EFA at `/opt/amazon/efa`. No extra library installation is needed when using these containers on EFA-enabled instances. - For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) - **Minimum required version:** v1.21.0 - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) diff --git a/tests/unittest/scripts/test_install_nixl.py b/tests/unittest/scripts/test_install_nixl.py index a127d96b4e86..a87e87da655d 100644 --- a/tests/unittest/scripts/test_install_nixl.py +++ b/tests/unittest/scripts/test_install_nixl.py @@ -27,7 +27,8 @@ def test_libfabric_path_requires_efa_development_files( dependency.parent.mkdir(parents=True, exist_ok=True) dependency.touch() - # Relocate the installation prefix without needing /opt write access. + # NGC PyTorch images bundle AWS EFA under /opt/amazon/efa. + # Relocate that prefix so the test does not need /opt write access. installer = tmp_path / "install_nixl.sh" installer.write_text(INSTALL_NIXL.read_text().replace("/opt/amazon/efa", str(efa_path))) (tmp_path / "github_auth.sh").write_text(":\n") From bb429c9ea0776d599687363a35e146ce335689ed Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Tue, 13 Oct 2026 20:09:06 +0800 Subject: [PATCH 06/27] Reuse bundled HPC-X hwloc for NIXL EFA builds Prefer the default MPI prefix's hwloc pkg-config metadata when the bundled EFA and hwloc development files are present. Preserve existing pkg-config search paths and system detection for other image layouts. Remove the added Ubuntu hwloc development package and update the setup guide to describe reusing the MPI stack's development files. Extend the CPU installer regression cases to cover hwloc selection and fallback. Signed-off-by: Yanchao Lu --- docker/common/install_base.sh | 1 - docker/common/install_nixl.sh | 8 ++++ examples/disaggregated/README.md | 8 ++-- tests/unittest/scripts/test_install_nixl.py | 50 ++++++++++++++++----- 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/docker/common/install_base.sh b/docker/common/install_base.sh index c9326c597b6d..20f9060f9e56 100644 --- a/docker/common/install_base.sh +++ b/docker/common/install_base.sh @@ -88,7 +88,6 @@ init_ubuntu() { libstdc++-14-dev \ libnuma1 \ libnuma-dev \ - libhwloc-dev \ python3-dev \ python3-pip \ python-is-python3 \ diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index 3d816fba02b6..63715b43cddb 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -12,6 +12,14 @@ if [[ ! -f "${LIBFABRIC_INSTALL_PATH}/include/rdma/fabric.h" || ! -f "${LIBFABRIC_INSTALL_PATH}/lib/libfabric.so" ]]; then LIBFABRIC_INSTALL_PATH="" fi +# Reuse the default MPI's hwloc in NGC PyTorch images for the EFA plugin. +HWLOC_INSTALL_PATH="/usr/local/mpi" +if [[ -n "${LIBFABRIC_INSTALL_PATH}" && + -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" && + -f "${HWLOC_INSTALL_PATH}/lib/libhwloc.so" && + -f "${HWLOC_INSTALL_PATH}/lib/pkgconfig/hwloc.pc" ]]; then + export PKG_CONFIG_PATH="${HWLOC_INSTALL_PATH}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" +fi CUDA_PATH="/usr/local/cuda" NIXL_VERSION="v1.4.0" NIXL_REPO="https://github.com/ai-dynamo/nixl.git" diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index f25eadfcd89b..6cdaceac83e6 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -57,9 +57,9 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) **hwloc** -- Ubuntu container builds install `libhwloc-dev` in `docker/common/install_base.sh`. For other base images, ensure the hwloc development files are installed before building the plugin. +- NGC PyTorch container builds reuse the hwloc development files bundled with HPC-X under `/usr/local/mpi`. When the bundled EFA and hwloc development files are present, `docker/common/install_nixl.sh` prepends `/usr/local/mpi/lib/pkgconfig` to `PKG_CONFIG_PATH`; no extra `libhwloc-dev` installation is needed for this configuration. - hwloc is used to understand the underlying architecture to optimize application performance -- **Suggested version:** 2.10.0 or newer +- For other base images or custom MPI installations, provide compatible hwloc development files and ensure their `hwloc.pc` is discoverable through `PKG_CONFIG_PATH`. **Network Hardware Requirements:** - Validated compatibility with AWS EFA (Elastic Fabric Adapter) @@ -77,8 +77,8 @@ Skip these steps if your container already includes the NIXL LIBFABRIC plugin fo 1. **Install libfabric dependencies:** - Follow the installation instructions from the links above based on your system -2. **Install hwloc:** - - Ubuntu container builds already include `libhwloc-dev`; install the equivalent development package if it is missing from your image +2. **Provide hwloc development files:** + - Reuse the hwloc bundled with your MPI installation when available, and ensure its `hwloc.pc` is discoverable through `PKG_CONFIG_PATH`. Otherwise, install a compatible hwloc development package before rebuilding NIXL. 3. **Reinstall NIXL after installing libfabric:** - After installing libfabric and hwloc, you must rebuild NIXL to generate the LIBFABRIC plugin against your custom path diff --git a/tests/unittest/scripts/test_install_nixl.py b/tests/unittest/scripts/test_install_nixl.py index a87e87da655d..97d6a4a13b9d 100644 --- a/tests/unittest/scripts/test_install_nixl.py +++ b/tests/unittest/scripts/test_install_nixl.py @@ -14,8 +14,17 @@ @pytest.mark.parametrize("with_header", (False, True)) @pytest.mark.parametrize("with_library", (False, True)) -def test_libfabric_path_requires_efa_development_files( - tmp_path: Path, with_header: bool, with_library: bool +@pytest.mark.parametrize( + "missing_hwloc", + (None, "all", "include/hwloc.h", "lib/libhwloc.so", "lib/pkgconfig/hwloc.pc"), +) +@pytest.mark.parametrize("pkg_config_path", (None, "/custom/lib/pkgconfig")) +def test_bundled_dependencies_require_development_files( + tmp_path: Path, + with_header: bool, + with_library: bool, + missing_hwloc: str | None, + pkg_config_path: str | None, ) -> None: efa_path = tmp_path / "efa" for present, name in ( @@ -27,10 +36,20 @@ def test_libfabric_path_requires_efa_development_files( dependency.parent.mkdir(parents=True, exist_ok=True) dependency.touch() - # NGC PyTorch images bundle AWS EFA under /opt/amazon/efa. - # Relocate that prefix so the test does not need /opt write access. + mpi_path = tmp_path / "mpi" + for name in ("include/hwloc.h", "lib/libhwloc.so", "lib/pkgconfig/hwloc.pc"): + if missing_hwloc not in ("all", name): + dependency = mpi_path / name + dependency.parent.mkdir(parents=True, exist_ok=True) + dependency.touch() + + # Relocate NGC PyTorch's bundled EFA and MPI prefixes without root access. installer = tmp_path / "install_nixl.sh" - installer.write_text(INSTALL_NIXL.read_text().replace("/opt/amazon/efa", str(efa_path))) + installer.write_text( + INSTALL_NIXL.read_text() + .replace("/opt/amazon/efa", str(efa_path)) + .replace("/usr/local/mpi", str(mpi_path)) + ) (tmp_path / "github_auth.sh").write_text(":\n") meson_args = tmp_path / "meson_args" command = r""" @@ -39,18 +58,23 @@ def test_libfabric_path_requires_efa_development_files( find() { printf '%s\n' /usr/local/cuda/lib64/libcuda.so.1; } meson() { printf '%s\n' "$@" > "$MESON_ARGS" + printf '%s' "${PKG_CONFIG_PATH-}" > "$MESON_PKG_CONFIG_PATH" exit 0 } source "$INSTALL_NIXL" """ + env = { + "PATH": os.environ["PATH"], + "INSTALL_NIXL": str(installer), + "MESON_ARGS": str(meson_args), + "MESON_PKG_CONFIG_PATH": str(tmp_path / "meson_pkg_config_path"), + } + if pkg_config_path is not None: + env["PKG_CONFIG_PATH"] = pkg_config_path result = subprocess.run( ["bash", "-c", command], cwd=tmp_path, - env={ - "PATH": os.environ["PATH"], - "INSTALL_NIXL": str(installer), - "MESON_ARGS": str(meson_args), - }, + env=env, capture_output=True, text=True, timeout=30, @@ -62,3 +86,9 @@ def test_libfabric_path_requires_efa_development_files( assert [arg for arg in args if arg.startswith("-Dlibfabric_path=")] == [ f"-Dlibfabric_path={expected_path}" ] + expected_pkg_config_path = pkg_config_path or "" + if with_header and with_library and missing_hwloc is None: + expected_pkg_config_path = str(mpi_path / "lib/pkgconfig") + if pkg_config_path: + expected_pkg_config_path += f":{pkg_config_path}" + assert (tmp_path / "meson_pkg_config_path").read_text() == expected_pkg_config_path From 9ac2c25f03749b4f5c9ebd1f531a0b1e46e64d2c Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Tue, 13 Oct 2026 20:28:15 +0800 Subject: [PATCH 07/27] Clarify bundled hwloc and libfabric fallback comments Signed-off-by: Yanchao Lu --- docker/common/install_nixl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index 63715b43cddb..fe9812d4890f 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -12,7 +12,7 @@ if [[ ! -f "${LIBFABRIC_INSTALL_PATH}/include/rdma/fabric.h" || ! -f "${LIBFABRIC_INSTALL_PATH}/lib/libfabric.so" ]]; then LIBFABRIC_INSTALL_PATH="" fi -# Reuse the default MPI's hwloc in NGC PyTorch images for the EFA plugin. +# Reuse HPC-X's hwloc via NGC PyTorch's default Open MPI prefix for the EFA plugin. HWLOC_INSTALL_PATH="/usr/local/mpi" if [[ -n "${LIBFABRIC_INSTALL_PATH}" && -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" && @@ -50,6 +50,7 @@ fi CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH +# An empty libfabric_path uses optional system detection; no match skips the plugin. meson setup builddir \ -Ducx_path=$UCX_INSTALL_PATH \ -Dlibfabric_path="${LIBFABRIC_INSTALL_PATH}" \ From 8299d9a7a5a5a2919933a77ee34258e6ae454d55 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Tue, 13 Oct 2026 20:37:36 +0800 Subject: [PATCH 08/27] Explain the HPC-X origin of the default Open MPI Signed-off-by: Yanchao Lu --- docker/common/install_nixl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index fe9812d4890f..c96fa1f38fb6 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -12,7 +12,8 @@ if [[ ! -f "${LIBFABRIC_INSTALL_PATH}/include/rdma/fabric.h" || ! -f "${LIBFABRIC_INSTALL_PATH}/lib/libfabric.so" ]]; then LIBFABRIC_INSTALL_PATH="" fi -# Reuse HPC-X's hwloc via NGC PyTorch's default Open MPI prefix for the EFA plugin. +# NGC PyTorch's default Open MPI comes from HPC-X and is exposed at /usr/local/mpi. +# Reuse the hwloc bundled with that MPI installation for the EFA plugin. HWLOC_INSTALL_PATH="/usr/local/mpi" if [[ -n "${LIBFABRIC_INSTALL_PATH}" && -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" && From b19aa480d7648625ad7a1b7666febc66b1b71591 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Tue, 13 Oct 2026 23:19:34 +0800 Subject: [PATCH 09/27] Use staging images to validate NIXL LIBFABRIC in CI Signed-off-by: Yanchao Lu --- jenkins/current_image_tags.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 428f962e04f1..d55a3b44a36f 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm -LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.09-py3-x86_64-ubuntu24.04-skip-tritondevel-202610130652-20175 -LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.09-py3-sbsa-ubuntu24.04-skip-tritondevel-202610130652-20175 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.4.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202610130652-20175 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.4.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202610130652-20175 -LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.4.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202610130652-20175 +LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-tritondevel-torch_skip-6a24e35-github-pr-19783-63911 +LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-tritondevel-torch_skip-6a24e35-github-pr-19783-63911 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py310-6a24e35-github-pr-19783-63911 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py312-6a24e35-github-pr-19783-63911 +LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-ubuntu24-torch_skip-py312-6a24e35-github-pr-19783-63911 From 9302e61f61e6f53df0d8407923b274f748137e29 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Wed, 14 Oct 2026 21:34:17 +0800 Subject: [PATCH 10/27] Fix NIXL LIBFABRIC context selection for CUDA VMM Signed-off-by: Yanchao Lu --- docker/common/install_nixl.sh | 6 ++++- .../nixl-v1.4.0-cuda-vmm-context.patch | 26 +++++++++++++++++++ tests/unittest/scripts/test_install_nixl.py | 12 ++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index c96fa1f38fb6..591667c4c8d0 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -1,8 +1,11 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 set -ex +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Authenticate the github.com clone below; no-op when no token is available. -source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh" +source "${SCRIPT_DIR}/github_auth.sh" GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" @@ -40,6 +43,7 @@ pip3 install meson ninja pybind11 setuptools git clone --depth 1 -b ${NIXL_VERSION} ${NIXL_REPO} cd nixl +git apply "${SCRIPT_DIR}/patches/nixl-v1.4.0-cuda-vmm-context.patch" CUDA_SO_PATH=$(find "/usr/local" -name "libcuda.so.1" 2>/dev/null | head -n1) diff --git a/docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch b/docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch new file mode 100644 index 000000000000..7995e0d97477 --- /dev/null +++ b/docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch @@ -0,0 +1,26 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Subject: Fall back to the device primary context for CUDA VMM allocations +Upstream: https://github.com/ai-dynamo/nixl/pull/2425 + +--- a/src/plugins/libfabric/libfabric_backend.cpp ++++ b/src/plugins/libfabric/libfabric_backend.cpp +@@ -267,6 +267,8 @@ + if (ret) return ret; + if (!is_dev) return 0; + if (dev != expected_dev) return -1; ++ // Context-independent allocations need the per-device primary-context path. ++ if (ctx == nullptr) return -1; + + if (pthrCudaCtx_) { + if (pthrCudaCtx_ != ctx) return -1; +@@ -900,7 +902,7 @@ + if (use_cuda_addr_wa) { + bool need_restart; + if (vramUpdateCtx((void *)mem.addr, mem.devId, need_restart)) { +- NIXL_INFO << "Multi-GPU detected (device " << mem.devId ++ NIXL_INFO << "CUDA pointer context unavailable or incompatible (device " << mem.devId + << "), using cudaSetDevice fallback"; + { + const std::lock_guard lock(cuda_ctx_mutex_); diff --git a/tests/unittest/scripts/test_install_nixl.py b/tests/unittest/scripts/test_install_nixl.py index 97d6a4a13b9d..39962f76e1c3 100644 --- a/tests/unittest/scripts/test_install_nixl.py +++ b/tests/unittest/scripts/test_install_nixl.py @@ -54,7 +54,13 @@ def test_bundled_dependencies_require_development_files( meson_args = tmp_path / "meson_args" command = r""" pip3() { :; } -git() { mkdir nixl; } +git() { + case "$1" in + clone) mkdir nixl ;; + apply) printf '%s\n' "$2" > "$NIXL_PATCH" ;; + *) return 1 ;; + esac +} find() { printf '%s\n' /usr/local/cuda/lib64/libcuda.so.1; } meson() { printf '%s\n' "$@" > "$MESON_ARGS" @@ -68,6 +74,7 @@ def test_bundled_dependencies_require_development_files( "INSTALL_NIXL": str(installer), "MESON_ARGS": str(meson_args), "MESON_PKG_CONFIG_PATH": str(tmp_path / "meson_pkg_config_path"), + "NIXL_PATCH": str(tmp_path / "nixl_patch"), } if pkg_config_path is not None: env["PKG_CONFIG_PATH"] = pkg_config_path @@ -81,6 +88,9 @@ def test_bundled_dependencies_require_development_files( ) assert result.returncode == 0, result.stderr + assert (tmp_path / "nixl_patch").read_text().strip() == str( + tmp_path / "patches/nixl-v1.4.0-cuda-vmm-context.patch" + ) args = meson_args.read_text().splitlines() expected_path = str(efa_path) if with_header and with_library else "" assert [arg for arg in args if arg.startswith("-Dlibfabric_path=")] == [ From 3c5b4ce76eb8facd76cab0e469142db01e29f94f Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Wed, 14 Oct 2026 21:54:37 +0800 Subject: [PATCH 11/27] Backport AWS libfabric CUDA VMM synchronization support Signed-off-by: Yanchao Lu --- docker/Dockerfile.multi | 1 + docker/common/install_libfabric.sh | 72 +++++ .../libfabric-2.4.0amzn5.0-cuda-vmm.patch | 255 ++++++++++++++++++ examples/disaggregated/README.md | 11 +- .../scripts/test_install_libfabric.py | 116 ++++++++ 5 files changed, 451 insertions(+), 4 deletions(-) create mode 100644 docker/common/install_libfabric.sh create mode 100644 docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch create mode 100644 tests/unittest/scripts/test_install_libfabric.py diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index c6712f3c8b4e..6fb5c805d79d 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -96,6 +96,7 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ --mount=type=secret,id=github_clone_token \ . /opt/docker/common/github_auth.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_ucx.sh && \ + bash /opt/docker/common/install_libfabric.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_nixl.sh && \ bash /opt/docker/common/install_etcd.sh && \ rm -rf /root/.cache/uv/archive-v0 && \ diff --git a/docker/common/install_libfabric.sh b/docker/common/install_libfabric.sh new file mode 100644 index 000000000000..39d36c72e4a1 --- /dev/null +++ b/docker/common/install_libfabric.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EFA_PATH="/opt/amazon/efa" +LIBFABRIC_VERSION="2.4.0amzn5.0" + +# Only patch the AWS EFA installation bundled with NGC PyTorch images. +if [[ ! -f "${EFA_PATH}/include/rdma/fabric.h" || ! -f "${EFA_PATH}/lib/libfabric.so" ]]; then + exit 0 +fi +installed_version=$(PKG_CONFIG_LIBDIR="${EFA_PATH}/lib/pkgconfig" PKG_CONFIG_PATH= pkg-config --modversion libfabric) +if [[ "${installed_version}" != "${LIBFABRIC_VERSION}" ]]; then + echo "Revalidate the CUDA VMM backport for AWS libfabric ${installed_version}." >&2 + exit 1 +fi + +BUILD_DIR=$(mktemp -d) +trap 'rm -rf "${BUILD_DIR}"' EXIT +cd "${BUILD_DIR}" + +providers() { + env -u FI_PROVIDER -u FI_PROVIDER_PATH -u LD_PRELOAD LD_LIBRARY_PATH="$1/lib" \ + "$1/bin/fi_info" -l | awk '/:$/ { print }' | LC_ALL=C sort +} +providers "${EFA_PATH}" > providers.before + +curl --fail --location --retry 3 \ + "https://codeload.github.com/aws/libfabric/tar.gz/refs/tags/v${LIBFABRIC_VERSION}" -o libfabric.tar.gz +echo '2b0f027fc4ac1f0731b1c4dca786d9c2d0205ddc9d5f660a5685e8b58fc4213b libfabric.tar.gz' | sha256sum -c - +tar -xzf libfabric.tar.gz + +# Neuron is loaded dynamically by the original library; retain that support. +# These are build-only headers, not a replacement Neuron runtime. +NEURON_COMMIT=e27a3986fa8625d4103a7ce611ee0c85ae79979a +mkdir -p neuron/include/nrt neuron/include/ndl +while read -r checksum header; do + curl --fail --location --retry 3 \ + "https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/${NEURON_COMMIT}/src/libnrt/include/${header}" \ + -o "neuron/include/${header}" + echo "${checksum} neuron/include/${header}" | sha256sum -c - +done <<'HEADERS' +1890d030c4628e5f83413bd30f5a52986e55040e853cbeb0acd064f827886dcf nrt/nrt.h +701bffe3b5c2277deef3df96e9d0ac08d45e819812148196f47e1bf458e218a1 nrt/nrt_experimental.h +19d1b61e30610022df94103f8e19e4cb8948acc6a334401d1d2f8da608a2e995 nrt/nrt_status.h +7f34780056a43c05f6c90b54eac87cd3c1c5b92740a9f06436e106fa0383cff1 ndl/neuron_driver_shared_tensor_batch_op.h +HEADERS + +cd "libfabric-${LIBFABRIC_VERSION}" +patch --batch -p1 < "${SCRIPT_DIR}/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch" +mkdir -p /third-party-source +tar -czf "/third-party-source/libfabric-${LIBFABRIC_VERSION}-cuda-vmm.tar.gz" . +./autogen.sh +# Preserve the bundled providers and dynamic accelerator support. +./configure --prefix="${EFA_PATH}" --libdir="${EFA_PATH}/lib" --disable-static \ + --enable-only --enable-efa --enable-rxm --enable-rxd --enable-shm \ + --enable-udp --enable-tcp --enable-sockets --enable-sm2 --enable-mrail \ + --enable-perf --enable-trace --enable-monitor --enable-hook_debug \ + --enable-hook_hmem --enable-dmabuf_peer_mem \ + --enable-cuda-dlopen --with-cuda=/usr/local/cuda \ + --enable-gdrcopy-dlopen --with-gdrcopy=/usr --with-neuron="${BUILD_DIR}/neuron" +for feature in HAVE_CUDA HAVE_CUDA_CTX_SYNC_MEMOPS HAVE_GDRCOPY HAVE_NEURON; do + grep -q "^#define ${feature} 1$" config.h +done +make -j"$(nproc)" +make DESTDIR="${BUILD_DIR}/install" install +providers "${BUILD_DIR}/install${EFA_PATH}" > "${BUILD_DIR}/providers.after" +diff -u "${BUILD_DIR}/providers.before" "${BUILD_DIR}/providers.after" +cp -a "${BUILD_DIR}/install${EFA_PATH}/." "${EFA_PATH}/" +ldconfig diff --git a/docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch b/docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch new file mode 100644 index 000000000000..9ad400bc11bf --- /dev/null +++ b/docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch @@ -0,0 +1,255 @@ +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. +SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only + +Backport CUDA VMM synchronization support to AWS libfabric 2.4.0amzn5.0: +https://github.com/ofiwg/libfabric/commit/2c7d11a50a3db03d26ad8260f8141621eee495f7 +https://github.com/ofiwg/libfabric/commit/790f0fb944f4cbeeb4381c99db6ba827ce5ded27 + +The EFA MR device field is peer.device in this release. Keep the optional CUDA +symbol handling and preserve existing primary-context flags. Remove this +backport when the bundled AWS release includes both upstream fixes. + +--- a/configure.ac ++++ b/configure.ac +@@ -737,6 +737,7 @@ + cuda_dlopen=0 + have_cuda_dmabuf=0 + have_cuda_dmabuf_mapping_type_pcie=0 ++have_cuda_ctx_sync_memops=0 + AC_ARG_ENABLE([cuda-dlopen], + [AS_HELP_STRING([--enable-cuda-dlopen], + [Enable dlopen of CUDA libraries @<:@default=no@:>@]) +@@ -793,6 +794,12 @@ + [have_cuda_dmabuf_mapping_type_pcie=0], + [[#include ]]) + ]) ++ ++AS_IF([test "$have_cuda" = "1"], [ ++ AC_CHECK_DECL([CU_CTX_SYNC_MEMOPS], ++ [have_cuda_ctx_sync_memops=1], [], [[#include ]]) ++]) ++AC_DEFINE_UNQUOTED([HAVE_CUDA_CTX_SYNC_MEMOPS], [$have_cuda_ctx_sync_memops], [CUDA context-wide sync memops support]) + + AC_DEFINE_UNQUOTED([ENABLE_CUDA_DLOPEN], [$cuda_dlopen], [dlopen CUDA libraries]) + +--- a/include/ofi_hmem.h ++++ b/include/ofi_hmem.h +@@ -71,6 +71,12 @@ + cudaError_t ofi_cudaHostUnregister(void *ptr); + cudaError_t ofi_cudaMalloc(void **ptr, size_t size); + cudaError_t ofi_cudaFree(void *ptr); ++ ++#if HAVE_CUDA_CTX_SYNC_MEMOPS ++CUresult ofi_cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int *flags, ++ int *active); ++CUresult ofi_cuDevicePrimaryCtxSetFlags(CUdevice dev, unsigned int flags); ++#endif + + #endif /* HAVE_CUDA */ + +--- a/prov/efa/src/efa_hmem.c ++++ b/prov/efa/src/efa_hmem.c +@@ -8,6 +8,46 @@ + #include "rdm/efa_rdm_pkt_type.h" + + struct efa_hmem_info g_efa_hmem_info[OFI_HMEM_MAX]; ++ ++int efa_hmem_set_sync_memops(void *ptr, uint64_t device) ++{ ++#if HAVE_CUDA_CTX_SYNC_MEMOPS ++ CUresult cu_result; ++ CUdevice cu_device = (CUdevice) device; ++ unsigned int ctx_flags; ++ int active; ++ int ret; ++ ++ ret = cuda_set_sync_memops(ptr); ++ if (ret != -FI_EOPNOTSUPP) ++ return ret; ++ ++ cu_result = ofi_cuDevicePrimaryCtxGetState(cu_device, &ctx_flags, &active); ++ if (cu_result != CUDA_SUCCESS) { ++ EFA_WARN(FI_LOG_MR, ++ "Failed to get CUDA device %d primary context flags: %d\n", ++ cu_device, (int) cu_result); ++ return -FI_EINVAL; ++ } ++ ++ if (ctx_flags & CU_CTX_SYNC_MEMOPS) ++ return FI_SUCCESS; ++ ++ cu_result = ofi_cuDevicePrimaryCtxSetFlags(cu_device, ctx_flags | CU_CTX_SYNC_MEMOPS); ++ if (cu_result == CUDA_SUCCESS) ++ return FI_SUCCESS; ++ ++ EFA_WARN(FI_LOG_MR, ++ "Failed to set CUDA device %d primary context flag " ++ "CU_CTX_SYNC_MEMOPS: %d\n", ++ cu_device, (int) cu_result); ++ return -FI_EINVAL; ++#else ++ (void) device; ++ return cuda_set_sync_memops(ptr); ++#endif ++} ++ + + // TODO double-check for ROCr + #if HAVE_CUDA || HAVE_NEURON || HAVE_ROCR +--- a/prov/efa/src/efa_hmem.h ++++ b/prov/efa/src/efa_hmem.h +@@ -63,6 +63,7 @@ + + int efa_hmem_validate_p2p_opt(enum fi_hmem_iface iface, int p2p_opt, uint32_t api_version); + int efa_hmem_info_initialize(); ++int efa_hmem_set_sync_memops(void *ptr, uint64_t device); + int efa_copy_from_hmem(void *desc, void *dest, const void *src, size_t size); + int efa_copy_to_hmem(void *desc, void *dest, const void *src, size_t size); + ssize_t efa_copy_from_hmem_iov(void **desc, char *buff, size_t buff_size, const struct iovec *hmem_iov, size_t iov_count); +--- a/prov/efa/src/rdm/efa_rdm_ep.h ++++ b/prov/efa/src/rdm/efa_rdm_ep.h +@@ -511,7 +511,8 @@ + for (i = 0; i < num_desc; i++) { + efa_mr = (struct efa_mr *) desc[i]; + if (efa_mr && efa_mr->needs_sync) { +- err = cuda_set_sync_memops(iov[i].iov_base); ++ err = efa_hmem_set_sync_memops( ++ iov[i].iov_base, efa_mr->peer.device); + if (err) { + EFA_WARN(FI_LOG_MR, + "Unable to set memops for " +@@ -539,7 +540,8 @@ + for (i = 0; i < num_desc; i++) { + efa_mr = (struct efa_mr *) desc[i]; + if (efa_mr && efa_mr->needs_sync) { +- err = cuda_set_sync_memops(ioc[i].addr); ++ err = efa_hmem_set_sync_memops( ++ ioc[i].addr, efa_mr->peer.device); + if (err) { + EFA_WARN(FI_LOG_MR, + "Unable to set memops for " +--- a/src/hmem_cuda.c ++++ b/src/hmem_cuda.c +@@ -50,6 +50,14 @@ + #endif + + /* ++ * Explicitly declare cuDevicePrimaryCtxSetFlags_v2 so its dynamic symbol name ++ * is stable regardless of the cuDevicePrimaryCtxSetFlags macro definition in ++ * the CUDA headers. ++ */ ++CUresult CUDAAPI cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, ++ unsigned int flags); ++ ++/* + * Convenience higher-order macros for enumerating CUDA driver/runtime and + * NVML API function names + */ +@@ -59,6 +67,14 @@ + _(cuMemGetHandleForAddressRange) + #else + #define CUDA_DRIVER_DMABUF_FUNCS_DEF(_) ++#endif ++ ++#if HAVE_CUDA_CTX_SYNC_MEMOPS ++#define CUDA_DRIVER_OPTIONAL_FUNCS_DEF(_) \ ++ _(cuDevicePrimaryCtxGetState) \ ++ _(cuDevicePrimaryCtxSetFlags_v2) ++#else ++#define CUDA_DRIVER_OPTIONAL_FUNCS_DEF(_) + #endif + + #define CUDA_DRIVER_FUNCS_DEF(_) \ +@@ -147,6 +163,13 @@ + CUresult (*cuDeviceGetAttribute)(int* pi, + CUdevice_attribute attrib, CUdevice dev); + CUresult (*cuDeviceGet)(CUdevice* device, int ordinal); ++#if HAVE_CUDA_CTX_SYNC_MEMOPS ++ CUresult (*cuDevicePrimaryCtxGetState)(CUdevice dev, ++ unsigned int *flags, ++ int *active); ++ CUresult (*cuDevicePrimaryCtxSetFlags_v2)(CUdevice dev, ++ unsigned int flags); ++#endif + cudaError_t (*cudaHostRegister)(void *ptr, size_t size, + unsigned int flags); + cudaError_t (*cudaHostUnregister)(void *ptr); +@@ -167,6 +190,7 @@ + #define CUDA_OPS_INIT(sym) .sym = sym, + = { + CUDA_DRIVER_FUNCS_DEF(CUDA_OPS_INIT) ++ CUDA_DRIVER_OPTIONAL_FUNCS_DEF(CUDA_OPS_INIT) + CUDA_RUNTIME_FUNCS_DEF(CUDA_OPS_INIT) + NVML_FUNCS_DEF(CUDA_OPS_INIT) + } +@@ -252,7 +276,8 @@ + * the data consistency for CUDA IPC. + * + * @param ptr the cuda ptr +- * @return int 0 on success, -FI_EINVAL on failure. ++ * @return int 0 on success, -FI_EOPNOTSUPP when the pointer attribute is ++ * unsupported, or -FI_EINVAL on other failures. + */ + int cuda_set_sync_memops(void *ptr) + { +@@ -266,6 +291,9 @@ + (CUdeviceptr) ptr); + if (cu_result == CUDA_SUCCESS) + return FI_SUCCESS; ++ ++ if (cu_result == CUDA_ERROR_NOT_SUPPORTED) ++ return -FI_EOPNOTSUPP; + + ofi_cuGetErrorName(cu_result, &cu_error_name); + ofi_cuGetErrorString(cu_result, &cu_error_str); +@@ -274,6 +302,25 @@ + cu_error_name, cu_error_str); + return -FI_EINVAL; + } ++ ++#if HAVE_CUDA_CTX_SYNC_MEMOPS ++CUresult ofi_cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int *flags, ++ int *active) ++{ ++ if (!cuda_ops.cuDevicePrimaryCtxGetState) ++ return CUDA_ERROR_NOT_SUPPORTED; ++ ++ return cuda_ops.cuDevicePrimaryCtxGetState(dev, flags, active); ++} ++ ++CUresult ofi_cuDevicePrimaryCtxSetFlags(CUdevice dev, unsigned int flags) ++{ ++ if (!cuda_ops.cuDevicePrimaryCtxSetFlags_v2) ++ return CUDA_ERROR_NOT_SUPPORTED; ++ ++ return cuda_ops.cuDevicePrimaryCtxSetFlags_v2(dev, flags); ++} ++#endif + + CUresult ofi_cuMemGetAddressRange(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) + { +@@ -474,6 +521,15 @@ + #define CUDA_RUNTIME_FUNCS_DLOPEN(sym) CUDA_FUNCS_DLOPEN(runtime, sym) + #define NVML_LIB_FUNCS_DLOPEN(sym) CUDA_FUNCS_DLOPEN(nvml, sym) + ++#define CUDA_DRIVER_OPTIONAL_FUNCS_DLOPEN(sym) \ ++ do { \ ++ cuda_ops.sym = dlsym(cuda_attr.driver_handle, #sym); \ ++ if (!cuda_ops.sym) \ ++ FI_INFO(&core_prov, FI_LOG_CORE, \ ++ "Optional CUDA symbol " #sym \ ++ " is unavailable\n"); \ ++ } while (0); ++ + static int cuda_hmem_dl_init(void) + { + #if ENABLE_CUDA_DLOPEN +@@ -501,6 +557,7 @@ + } + + CUDA_DRIVER_FUNCS_DEF(CUDA_DRIVER_FUNCS_DLOPEN) ++ CUDA_DRIVER_OPTIONAL_FUNCS_DEF(CUDA_DRIVER_OPTIONAL_FUNCS_DLOPEN) + CUDA_RUNTIME_FUNCS_DEF(CUDA_RUNTIME_FUNCS_DLOPEN) + if (cuda_attr.nvml_handle) { + NVML_FUNCS_DEF(NVML_LIB_FUNCS_DLOPEN) diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index 6cdaceac83e6..7957652a5963 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -44,6 +44,8 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX **Note:** Containers built from NGC PyTorch base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). With these containers on EFA-enabled instances, no extra library installation is needed — set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. +For the CUDA VMM allocations used by KV cache V2, the NGC PyTorch 26.09 build also patches bundled AWS libfabric 2.4.0amzn5.0 and NIXL 1.4.0. These fixes handle context-independent allocations without requiring `NIXL_DISABLE_CUDA_ADDR_WA`. The libfabric rebuild retains the bundled provider set and CUDA, GDRCopy and Neuron support. A different bundled libfabric version requires revalidating the backport in `docker/common/install_libfabric.sh` before building. + #### Prerequisites ##### For LIBFABRIC Backend @@ -54,6 +56,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX - Bundled in NGC PyTorch base images with AWS EFA at `/opt/amazon/efa`. No extra library installation is needed when using these containers on EFA-enabled instances. - For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) - **Minimum required version:** v1.21.0 +- CUDA VMM additionally requires the [libfabric synchronization fixes](https://github.com/ofiwg/libfabric/issues/12853) and the [NIXL allocation-context fix](https://github.com/ai-dynamo/nixl/pull/2425). The minimum version alone does not guarantee VMM support; custom installations must include these fixes or an upstream release containing them. - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) **hwloc** @@ -93,7 +96,7 @@ If you have a pre-compiled `libplugin_LIBFABRIC.so` that matches your NIXL versi 1. Place the plugin file in a directory of your choice 2. Set the environment variable to point to the plugin directory: ```bash - export NIXL_PLUGINS_DIR=/path/to/plugin/directory + export NIXL_PLUGIN_DIR=/path/to/plugin/directory export TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC ``` 3. Ensure the plugin was built with the same NIXL version as in your container @@ -147,7 +150,7 @@ CUDA_VISIBLE_DEVICES=2 trtllm-serve TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ export TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC # If using pre-compiled plugin: -# export NIXL_PLUGINS_DIR=/path/to/plugin/directory +# export NIXL_PLUGIN_DIR=/path/to/plugin/directory # For AWS EFA (optional): # export FI_PROVIDER=efa @@ -184,7 +187,7 @@ CUDA_VISIBLE_DEVICES=2 trtllm-serve TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ - Standard UCX environment variables apply **For LIBFABRIC backend:** -- `NIXL_PLUGINS_DIR`: Directory containing the NIXL LIBFABRIC plugin (`libplugin_LIBFABRIC.so`) if using pre-compiled plugin +- `NIXL_PLUGIN_DIR`: Directory containing the NIXL LIBFABRIC plugin (`libplugin_LIBFABRIC.so`) if using pre-compiled plugin - `FI_PROVIDER`: Specifies the libfabric provider to use (e.g., `efa` for AWS EFA) - `FI_EFA_USE_DEVICE_RDMA`: Set to `1` to enable GPU Direct RDMA on AWS EFA (if supported) - `FI_LOG_LEVEL`: Controls libfabric logging verbosity (e.g., `warn`, `info`, `debug`) @@ -602,4 +605,4 @@ A: Check that the container includes the NIXL LIBFABRIC plugin and the dependenc 1. **Missing plugin**: Older or custom images without libfabric development files at build time may not include the plugin. Follow [LIBFABRIC Backend Setup](#libfabric-backend-setup) to install it. 2. **EFA devices unavailable**: If using EFA, confirm the instance supports it and the devices are passed through to the container. 3. **Custom libfabric version mismatch**: If you rebuilt NIXL against a different libfabric installation, confirm it meets the minimum required version (v1.21.0) and that the plugin was built against that same libfabric. -4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGINS_DIR` to point to its directory. +4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGIN_DIR` to point to its directory. diff --git a/tests/unittest/scripts/test_install_libfabric.py b/tests/unittest/scripts/test_install_libfabric.py new file mode 100644 index 000000000000..d75ab859bee9 --- /dev/null +++ b/tests/unittest/scripts/test_install_libfabric.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.cpu_only +INSTALLER = Path(__file__).resolve().parents[3] / "docker/common/install_libfabric.sh" + + +def run_installer(tmp_path: Path, *, present: bool = True, **overrides: str): + efa = tmp_path / "efa" + if present: + for filename in ("include/rdma/fabric.h", "lib/libfabric.so"): + path = efa / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("original") + binary = efa / "bin/fi_info" + binary.parent.mkdir() + binary.write_text("#!/bin/sh\nprintf 'efa:\\nshm:\\n'\n") + binary.chmod(0o755) + installer = tmp_path / "install_libfabric.sh" + installer.write_text( + INSTALLER.read_text() + .replace("/opt/amazon/efa", str(efa)) + .replace("/third-party-source", str(tmp_path / "source")) + ) + patch = tmp_path / "patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch" + patch.parent.mkdir() + patch.touch() + command = r""" +pkg-config() { printf '%s\n' "$VERSION"; } +curl() { touch "$CALLED"; } +sha256sum() { cat >/dev/null; return "$CHECKSUM_RC"; } +patch() { return "$PATCH_RC"; } +nproc() { echo 1; } +tar() { + if [ "$1" = -xzf ]; then + mkdir libfabric-2.4.0amzn5.0 + printf '#!/bin/sh\nexit 0\n' > libfabric-2.4.0amzn5.0/autogen.sh + cp libfabric-2.4.0amzn5.0/autogen.sh libfabric-2.4.0amzn5.0/configure + chmod +x libfabric-2.4.0amzn5.0/{autogen.sh,configure} + for feature in HAVE_CUDA HAVE_CUDA_CTX_SYNC_MEMOPS HAVE_GDRCOPY HAVE_NEURON; do + [ "$feature" = "$MISSING_FEATURE" ] || echo "#define $feature 1" + done > libfabric-2.4.0amzn5.0/config.h + fi +} +make() { + if [[ "${1:-}" = DESTDIR=* ]]; then + target="${1#DESTDIR=}$EFA_TEST_PATH" + mkdir -p "$target/lib" "$target/bin" + echo patched > "$target/lib/libfabric.so" + printf '#!/bin/sh\nprintf "%s"\n' "$NEW_PROVIDERS" > "$target/bin/fi_info" + chmod +x "$target/bin/fi_info" + fi +} +ldconfig() { touch "$INSTALLED"; } +source "$INSTALLER" +""" + env = { + "PATH": os.environ["PATH"], + "INSTALLER": str(installer), + "EFA_TEST_PATH": str(efa), + "CALLED": str(tmp_path / "download-started"), + "INSTALLED": str(tmp_path / "installed"), + "VERSION": "2.4.0amzn5.0", + "CHECKSUM_RC": "0", + "PATCH_RC": "0", + "MISSING_FEATURE": "", + "NEW_PROVIDERS": "shm:\\nefa:\\n", + **overrides, + } + return subprocess.run( + ["bash", "-c", command], env=env, capture_output=True, text=True, timeout=30 + ) + + +def test_no_bundled_efa_skips_download(tmp_path: Path) -> None: + result = run_installer(tmp_path, present=False) + assert result.returncode == 0, result.stderr + assert not (tmp_path / "download-started").exists() + + +def test_version_change_requires_revalidation(tmp_path: Path) -> None: + result = run_installer(tmp_path, VERSION="2.7.0amzn1.0") + assert result.returncode != 0 + assert "Revalidate" in result.stderr + assert not (tmp_path / "download-started").exists() + + +@pytest.mark.parametrize( + "failure", + [ + {"CHECKSUM_RC": "1"}, + {"PATCH_RC": "1"}, + {"MISSING_FEATURE": "HAVE_NEURON"}, + {"NEW_PROVIDERS": "efa:\\n"}, + ], +) +def test_failed_validation_preserves_original_library( + tmp_path: Path, failure: dict[str, str] +) -> None: + result = run_installer(tmp_path, **failure) + assert result.returncode != 0 + assert not (tmp_path / "installed").exists() + assert (tmp_path / "efa/lib/libfabric.so").read_text() == "original" + + +def test_matching_providers_allow_installation(tmp_path: Path) -> None: + result = run_installer(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "installed").exists() + assert (tmp_path / "efa/lib/libfabric.so").read_text().strip() == "patched" From 957ae54a83c9dccc154bd0aeb75dc7396305ea0b Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Wed, 14 Oct 2026 23:03:02 +0800 Subject: [PATCH 12/27] Test CUDA VMM fixes with freshly built staging images Signed-off-by: Yanchao Lu --- jenkins/current_image_tags.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index d55a3b44a36f..ec60c2c87b7d 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm -LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-tritondevel-torch_skip-6a24e35-github-pr-19783-63911 -LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-tritondevel-torch_skip-6a24e35-github-pr-19783-63911 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py310-6a24e35-github-pr-19783-63911 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py312-6a24e35-github-pr-19783-63911 -LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-ubuntu24-torch_skip-py312-6a24e35-github-pr-19783-63911 +LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-tritondevel-torch_skip-27399f8-github-pr-19783-64158 +LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-tritondevel-torch_skip-27399f8-github-pr-19783-64158 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py310-27399f8-github-pr-19783-64158 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py312-27399f8-github-pr-19783-64158 +LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-ubuntu24-torch_skip-py312-27399f8-github-pr-19783-64158 From f5cfc46d09d175cf4d543ee858a19f60d0f479d1 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Thu, 15 Oct 2026 08:57:22 +0800 Subject: [PATCH 13/27] Revert "Backport AWS libfabric CUDA VMM synchronization support" This reverts commit 3c5b4ce76eb8facd76cab0e469142db01e29f94f. Original pre-rebase commit: 27399f8ce7e5984a7648bef02a5e48adc7d47dfc. Use the libfabric provided by the official EFA Installer instead of rebuilding the runtime library in the TensorRT-LLM image. CUDA VMM synchronization support remains dependent on an official dependency update. Signed-off-by: Yanchao Lu --- docker/Dockerfile.multi | 1 - docker/common/install_libfabric.sh | 72 ----- .../libfabric-2.4.0amzn5.0-cuda-vmm.patch | 255 ------------------ examples/disaggregated/README.md | 11 +- .../scripts/test_install_libfabric.py | 116 -------- 5 files changed, 4 insertions(+), 451 deletions(-) delete mode 100644 docker/common/install_libfabric.sh delete mode 100644 docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch delete mode 100644 tests/unittest/scripts/test_install_libfabric.py diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index 6fb5c805d79d..c6712f3c8b4e 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -96,7 +96,6 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ --mount=type=secret,id=github_clone_token \ . /opt/docker/common/github_auth.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_ucx.sh && \ - bash /opt/docker/common/install_libfabric.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_nixl.sh && \ bash /opt/docker/common/install_etcd.sh && \ rm -rf /root/.cache/uv/archive-v0 && \ diff --git a/docker/common/install_libfabric.sh b/docker/common/install_libfabric.sh deleted file mode 100644 index 39d36c72e4a1..000000000000 --- a/docker/common/install_libfabric.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EFA_PATH="/opt/amazon/efa" -LIBFABRIC_VERSION="2.4.0amzn5.0" - -# Only patch the AWS EFA installation bundled with NGC PyTorch images. -if [[ ! -f "${EFA_PATH}/include/rdma/fabric.h" || ! -f "${EFA_PATH}/lib/libfabric.so" ]]; then - exit 0 -fi -installed_version=$(PKG_CONFIG_LIBDIR="${EFA_PATH}/lib/pkgconfig" PKG_CONFIG_PATH= pkg-config --modversion libfabric) -if [[ "${installed_version}" != "${LIBFABRIC_VERSION}" ]]; then - echo "Revalidate the CUDA VMM backport for AWS libfabric ${installed_version}." >&2 - exit 1 -fi - -BUILD_DIR=$(mktemp -d) -trap 'rm -rf "${BUILD_DIR}"' EXIT -cd "${BUILD_DIR}" - -providers() { - env -u FI_PROVIDER -u FI_PROVIDER_PATH -u LD_PRELOAD LD_LIBRARY_PATH="$1/lib" \ - "$1/bin/fi_info" -l | awk '/:$/ { print }' | LC_ALL=C sort -} -providers "${EFA_PATH}" > providers.before - -curl --fail --location --retry 3 \ - "https://codeload.github.com/aws/libfabric/tar.gz/refs/tags/v${LIBFABRIC_VERSION}" -o libfabric.tar.gz -echo '2b0f027fc4ac1f0731b1c4dca786d9c2d0205ddc9d5f660a5685e8b58fc4213b libfabric.tar.gz' | sha256sum -c - -tar -xzf libfabric.tar.gz - -# Neuron is loaded dynamically by the original library; retain that support. -# These are build-only headers, not a replacement Neuron runtime. -NEURON_COMMIT=e27a3986fa8625d4103a7ce611ee0c85ae79979a -mkdir -p neuron/include/nrt neuron/include/ndl -while read -r checksum header; do - curl --fail --location --retry 3 \ - "https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/${NEURON_COMMIT}/src/libnrt/include/${header}" \ - -o "neuron/include/${header}" - echo "${checksum} neuron/include/${header}" | sha256sum -c - -done <<'HEADERS' -1890d030c4628e5f83413bd30f5a52986e55040e853cbeb0acd064f827886dcf nrt/nrt.h -701bffe3b5c2277deef3df96e9d0ac08d45e819812148196f47e1bf458e218a1 nrt/nrt_experimental.h -19d1b61e30610022df94103f8e19e4cb8948acc6a334401d1d2f8da608a2e995 nrt/nrt_status.h -7f34780056a43c05f6c90b54eac87cd3c1c5b92740a9f06436e106fa0383cff1 ndl/neuron_driver_shared_tensor_batch_op.h -HEADERS - -cd "libfabric-${LIBFABRIC_VERSION}" -patch --batch -p1 < "${SCRIPT_DIR}/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch" -mkdir -p /third-party-source -tar -czf "/third-party-source/libfabric-${LIBFABRIC_VERSION}-cuda-vmm.tar.gz" . -./autogen.sh -# Preserve the bundled providers and dynamic accelerator support. -./configure --prefix="${EFA_PATH}" --libdir="${EFA_PATH}/lib" --disable-static \ - --enable-only --enable-efa --enable-rxm --enable-rxd --enable-shm \ - --enable-udp --enable-tcp --enable-sockets --enable-sm2 --enable-mrail \ - --enable-perf --enable-trace --enable-monitor --enable-hook_debug \ - --enable-hook_hmem --enable-dmabuf_peer_mem \ - --enable-cuda-dlopen --with-cuda=/usr/local/cuda \ - --enable-gdrcopy-dlopen --with-gdrcopy=/usr --with-neuron="${BUILD_DIR}/neuron" -for feature in HAVE_CUDA HAVE_CUDA_CTX_SYNC_MEMOPS HAVE_GDRCOPY HAVE_NEURON; do - grep -q "^#define ${feature} 1$" config.h -done -make -j"$(nproc)" -make DESTDIR="${BUILD_DIR}/install" install -providers "${BUILD_DIR}/install${EFA_PATH}" > "${BUILD_DIR}/providers.after" -diff -u "${BUILD_DIR}/providers.before" "${BUILD_DIR}/providers.after" -cp -a "${BUILD_DIR}/install${EFA_PATH}/." "${EFA_PATH}/" -ldconfig diff --git a/docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch b/docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch deleted file mode 100644 index 9ad400bc11bf..000000000000 --- a/docker/common/patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch +++ /dev/null @@ -1,255 +0,0 @@ -SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. -SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only - -Backport CUDA VMM synchronization support to AWS libfabric 2.4.0amzn5.0: -https://github.com/ofiwg/libfabric/commit/2c7d11a50a3db03d26ad8260f8141621eee495f7 -https://github.com/ofiwg/libfabric/commit/790f0fb944f4cbeeb4381c99db6ba827ce5ded27 - -The EFA MR device field is peer.device in this release. Keep the optional CUDA -symbol handling and preserve existing primary-context flags. Remove this -backport when the bundled AWS release includes both upstream fixes. - ---- a/configure.ac -+++ b/configure.ac -@@ -737,6 +737,7 @@ - cuda_dlopen=0 - have_cuda_dmabuf=0 - have_cuda_dmabuf_mapping_type_pcie=0 -+have_cuda_ctx_sync_memops=0 - AC_ARG_ENABLE([cuda-dlopen], - [AS_HELP_STRING([--enable-cuda-dlopen], - [Enable dlopen of CUDA libraries @<:@default=no@:>@]) -@@ -793,6 +794,12 @@ - [have_cuda_dmabuf_mapping_type_pcie=0], - [[#include ]]) - ]) -+ -+AS_IF([test "$have_cuda" = "1"], [ -+ AC_CHECK_DECL([CU_CTX_SYNC_MEMOPS], -+ [have_cuda_ctx_sync_memops=1], [], [[#include ]]) -+]) -+AC_DEFINE_UNQUOTED([HAVE_CUDA_CTX_SYNC_MEMOPS], [$have_cuda_ctx_sync_memops], [CUDA context-wide sync memops support]) - - AC_DEFINE_UNQUOTED([ENABLE_CUDA_DLOPEN], [$cuda_dlopen], [dlopen CUDA libraries]) - ---- a/include/ofi_hmem.h -+++ b/include/ofi_hmem.h -@@ -71,6 +71,12 @@ - cudaError_t ofi_cudaHostUnregister(void *ptr); - cudaError_t ofi_cudaMalloc(void **ptr, size_t size); - cudaError_t ofi_cudaFree(void *ptr); -+ -+#if HAVE_CUDA_CTX_SYNC_MEMOPS -+CUresult ofi_cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int *flags, -+ int *active); -+CUresult ofi_cuDevicePrimaryCtxSetFlags(CUdevice dev, unsigned int flags); -+#endif - - #endif /* HAVE_CUDA */ - ---- a/prov/efa/src/efa_hmem.c -+++ b/prov/efa/src/efa_hmem.c -@@ -8,6 +8,46 @@ - #include "rdm/efa_rdm_pkt_type.h" - - struct efa_hmem_info g_efa_hmem_info[OFI_HMEM_MAX]; -+ -+int efa_hmem_set_sync_memops(void *ptr, uint64_t device) -+{ -+#if HAVE_CUDA_CTX_SYNC_MEMOPS -+ CUresult cu_result; -+ CUdevice cu_device = (CUdevice) device; -+ unsigned int ctx_flags; -+ int active; -+ int ret; -+ -+ ret = cuda_set_sync_memops(ptr); -+ if (ret != -FI_EOPNOTSUPP) -+ return ret; -+ -+ cu_result = ofi_cuDevicePrimaryCtxGetState(cu_device, &ctx_flags, &active); -+ if (cu_result != CUDA_SUCCESS) { -+ EFA_WARN(FI_LOG_MR, -+ "Failed to get CUDA device %d primary context flags: %d\n", -+ cu_device, (int) cu_result); -+ return -FI_EINVAL; -+ } -+ -+ if (ctx_flags & CU_CTX_SYNC_MEMOPS) -+ return FI_SUCCESS; -+ -+ cu_result = ofi_cuDevicePrimaryCtxSetFlags(cu_device, ctx_flags | CU_CTX_SYNC_MEMOPS); -+ if (cu_result == CUDA_SUCCESS) -+ return FI_SUCCESS; -+ -+ EFA_WARN(FI_LOG_MR, -+ "Failed to set CUDA device %d primary context flag " -+ "CU_CTX_SYNC_MEMOPS: %d\n", -+ cu_device, (int) cu_result); -+ return -FI_EINVAL; -+#else -+ (void) device; -+ return cuda_set_sync_memops(ptr); -+#endif -+} -+ - - // TODO double-check for ROCr - #if HAVE_CUDA || HAVE_NEURON || HAVE_ROCR ---- a/prov/efa/src/efa_hmem.h -+++ b/prov/efa/src/efa_hmem.h -@@ -63,6 +63,7 @@ - - int efa_hmem_validate_p2p_opt(enum fi_hmem_iface iface, int p2p_opt, uint32_t api_version); - int efa_hmem_info_initialize(); -+int efa_hmem_set_sync_memops(void *ptr, uint64_t device); - int efa_copy_from_hmem(void *desc, void *dest, const void *src, size_t size); - int efa_copy_to_hmem(void *desc, void *dest, const void *src, size_t size); - ssize_t efa_copy_from_hmem_iov(void **desc, char *buff, size_t buff_size, const struct iovec *hmem_iov, size_t iov_count); ---- a/prov/efa/src/rdm/efa_rdm_ep.h -+++ b/prov/efa/src/rdm/efa_rdm_ep.h -@@ -511,7 +511,8 @@ - for (i = 0; i < num_desc; i++) { - efa_mr = (struct efa_mr *) desc[i]; - if (efa_mr && efa_mr->needs_sync) { -- err = cuda_set_sync_memops(iov[i].iov_base); -+ err = efa_hmem_set_sync_memops( -+ iov[i].iov_base, efa_mr->peer.device); - if (err) { - EFA_WARN(FI_LOG_MR, - "Unable to set memops for " -@@ -539,7 +540,8 @@ - for (i = 0; i < num_desc; i++) { - efa_mr = (struct efa_mr *) desc[i]; - if (efa_mr && efa_mr->needs_sync) { -- err = cuda_set_sync_memops(ioc[i].addr); -+ err = efa_hmem_set_sync_memops( -+ ioc[i].addr, efa_mr->peer.device); - if (err) { - EFA_WARN(FI_LOG_MR, - "Unable to set memops for " ---- a/src/hmem_cuda.c -+++ b/src/hmem_cuda.c -@@ -50,6 +50,14 @@ - #endif - - /* -+ * Explicitly declare cuDevicePrimaryCtxSetFlags_v2 so its dynamic symbol name -+ * is stable regardless of the cuDevicePrimaryCtxSetFlags macro definition in -+ * the CUDA headers. -+ */ -+CUresult CUDAAPI cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, -+ unsigned int flags); -+ -+/* - * Convenience higher-order macros for enumerating CUDA driver/runtime and - * NVML API function names - */ -@@ -59,6 +67,14 @@ - _(cuMemGetHandleForAddressRange) - #else - #define CUDA_DRIVER_DMABUF_FUNCS_DEF(_) -+#endif -+ -+#if HAVE_CUDA_CTX_SYNC_MEMOPS -+#define CUDA_DRIVER_OPTIONAL_FUNCS_DEF(_) \ -+ _(cuDevicePrimaryCtxGetState) \ -+ _(cuDevicePrimaryCtxSetFlags_v2) -+#else -+#define CUDA_DRIVER_OPTIONAL_FUNCS_DEF(_) - #endif - - #define CUDA_DRIVER_FUNCS_DEF(_) \ -@@ -147,6 +163,13 @@ - CUresult (*cuDeviceGetAttribute)(int* pi, - CUdevice_attribute attrib, CUdevice dev); - CUresult (*cuDeviceGet)(CUdevice* device, int ordinal); -+#if HAVE_CUDA_CTX_SYNC_MEMOPS -+ CUresult (*cuDevicePrimaryCtxGetState)(CUdevice dev, -+ unsigned int *flags, -+ int *active); -+ CUresult (*cuDevicePrimaryCtxSetFlags_v2)(CUdevice dev, -+ unsigned int flags); -+#endif - cudaError_t (*cudaHostRegister)(void *ptr, size_t size, - unsigned int flags); - cudaError_t (*cudaHostUnregister)(void *ptr); -@@ -167,6 +190,7 @@ - #define CUDA_OPS_INIT(sym) .sym = sym, - = { - CUDA_DRIVER_FUNCS_DEF(CUDA_OPS_INIT) -+ CUDA_DRIVER_OPTIONAL_FUNCS_DEF(CUDA_OPS_INIT) - CUDA_RUNTIME_FUNCS_DEF(CUDA_OPS_INIT) - NVML_FUNCS_DEF(CUDA_OPS_INIT) - } -@@ -252,7 +276,8 @@ - * the data consistency for CUDA IPC. - * - * @param ptr the cuda ptr -- * @return int 0 on success, -FI_EINVAL on failure. -+ * @return int 0 on success, -FI_EOPNOTSUPP when the pointer attribute is -+ * unsupported, or -FI_EINVAL on other failures. - */ - int cuda_set_sync_memops(void *ptr) - { -@@ -266,6 +291,9 @@ - (CUdeviceptr) ptr); - if (cu_result == CUDA_SUCCESS) - return FI_SUCCESS; -+ -+ if (cu_result == CUDA_ERROR_NOT_SUPPORTED) -+ return -FI_EOPNOTSUPP; - - ofi_cuGetErrorName(cu_result, &cu_error_name); - ofi_cuGetErrorString(cu_result, &cu_error_str); -@@ -274,6 +302,25 @@ - cu_error_name, cu_error_str); - return -FI_EINVAL; - } -+ -+#if HAVE_CUDA_CTX_SYNC_MEMOPS -+CUresult ofi_cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int *flags, -+ int *active) -+{ -+ if (!cuda_ops.cuDevicePrimaryCtxGetState) -+ return CUDA_ERROR_NOT_SUPPORTED; -+ -+ return cuda_ops.cuDevicePrimaryCtxGetState(dev, flags, active); -+} -+ -+CUresult ofi_cuDevicePrimaryCtxSetFlags(CUdevice dev, unsigned int flags) -+{ -+ if (!cuda_ops.cuDevicePrimaryCtxSetFlags_v2) -+ return CUDA_ERROR_NOT_SUPPORTED; -+ -+ return cuda_ops.cuDevicePrimaryCtxSetFlags_v2(dev, flags); -+} -+#endif - - CUresult ofi_cuMemGetAddressRange(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) - { -@@ -474,6 +521,15 @@ - #define CUDA_RUNTIME_FUNCS_DLOPEN(sym) CUDA_FUNCS_DLOPEN(runtime, sym) - #define NVML_LIB_FUNCS_DLOPEN(sym) CUDA_FUNCS_DLOPEN(nvml, sym) - -+#define CUDA_DRIVER_OPTIONAL_FUNCS_DLOPEN(sym) \ -+ do { \ -+ cuda_ops.sym = dlsym(cuda_attr.driver_handle, #sym); \ -+ if (!cuda_ops.sym) \ -+ FI_INFO(&core_prov, FI_LOG_CORE, \ -+ "Optional CUDA symbol " #sym \ -+ " is unavailable\n"); \ -+ } while (0); -+ - static int cuda_hmem_dl_init(void) - { - #if ENABLE_CUDA_DLOPEN -@@ -501,6 +557,7 @@ - } - - CUDA_DRIVER_FUNCS_DEF(CUDA_DRIVER_FUNCS_DLOPEN) -+ CUDA_DRIVER_OPTIONAL_FUNCS_DEF(CUDA_DRIVER_OPTIONAL_FUNCS_DLOPEN) - CUDA_RUNTIME_FUNCS_DEF(CUDA_RUNTIME_FUNCS_DLOPEN) - if (cuda_attr.nvml_handle) { - NVML_FUNCS_DEF(NVML_LIB_FUNCS_DLOPEN) diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index 7957652a5963..6cdaceac83e6 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -44,8 +44,6 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX **Note:** Containers built from NGC PyTorch base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). With these containers on EFA-enabled instances, no extra library installation is needed — set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. -For the CUDA VMM allocations used by KV cache V2, the NGC PyTorch 26.09 build also patches bundled AWS libfabric 2.4.0amzn5.0 and NIXL 1.4.0. These fixes handle context-independent allocations without requiring `NIXL_DISABLE_CUDA_ADDR_WA`. The libfabric rebuild retains the bundled provider set and CUDA, GDRCopy and Neuron support. A different bundled libfabric version requires revalidating the backport in `docker/common/install_libfabric.sh` before building. - #### Prerequisites ##### For LIBFABRIC Backend @@ -56,7 +54,6 @@ For the CUDA VMM allocations used by KV cache V2, the NGC PyTorch 26.09 build al - Bundled in NGC PyTorch base images with AWS EFA at `/opt/amazon/efa`. No extra library installation is needed when using these containers on EFA-enabled instances. - For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) - **Minimum required version:** v1.21.0 -- CUDA VMM additionally requires the [libfabric synchronization fixes](https://github.com/ofiwg/libfabric/issues/12853) and the [NIXL allocation-context fix](https://github.com/ai-dynamo/nixl/pull/2425). The minimum version alone does not guarantee VMM support; custom installations must include these fixes or an upstream release containing them. - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) **hwloc** @@ -96,7 +93,7 @@ If you have a pre-compiled `libplugin_LIBFABRIC.so` that matches your NIXL versi 1. Place the plugin file in a directory of your choice 2. Set the environment variable to point to the plugin directory: ```bash - export NIXL_PLUGIN_DIR=/path/to/plugin/directory + export NIXL_PLUGINS_DIR=/path/to/plugin/directory export TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC ``` 3. Ensure the plugin was built with the same NIXL version as in your container @@ -150,7 +147,7 @@ CUDA_VISIBLE_DEVICES=2 trtllm-serve TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ export TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC # If using pre-compiled plugin: -# export NIXL_PLUGIN_DIR=/path/to/plugin/directory +# export NIXL_PLUGINS_DIR=/path/to/plugin/directory # For AWS EFA (optional): # export FI_PROVIDER=efa @@ -187,7 +184,7 @@ CUDA_VISIBLE_DEVICES=2 trtllm-serve TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ - Standard UCX environment variables apply **For LIBFABRIC backend:** -- `NIXL_PLUGIN_DIR`: Directory containing the NIXL LIBFABRIC plugin (`libplugin_LIBFABRIC.so`) if using pre-compiled plugin +- `NIXL_PLUGINS_DIR`: Directory containing the NIXL LIBFABRIC plugin (`libplugin_LIBFABRIC.so`) if using pre-compiled plugin - `FI_PROVIDER`: Specifies the libfabric provider to use (e.g., `efa` for AWS EFA) - `FI_EFA_USE_DEVICE_RDMA`: Set to `1` to enable GPU Direct RDMA on AWS EFA (if supported) - `FI_LOG_LEVEL`: Controls libfabric logging verbosity (e.g., `warn`, `info`, `debug`) @@ -605,4 +602,4 @@ A: Check that the container includes the NIXL LIBFABRIC plugin and the dependenc 1. **Missing plugin**: Older or custom images without libfabric development files at build time may not include the plugin. Follow [LIBFABRIC Backend Setup](#libfabric-backend-setup) to install it. 2. **EFA devices unavailable**: If using EFA, confirm the instance supports it and the devices are passed through to the container. 3. **Custom libfabric version mismatch**: If you rebuilt NIXL against a different libfabric installation, confirm it meets the minimum required version (v1.21.0) and that the plugin was built against that same libfabric. -4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGIN_DIR` to point to its directory. +4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGINS_DIR` to point to its directory. diff --git a/tests/unittest/scripts/test_install_libfabric.py b/tests/unittest/scripts/test_install_libfabric.py deleted file mode 100644 index d75ab859bee9..000000000000 --- a/tests/unittest/scripts/test_install_libfabric.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import os -import subprocess -from pathlib import Path - -import pytest - -pytestmark = pytest.mark.cpu_only -INSTALLER = Path(__file__).resolve().parents[3] / "docker/common/install_libfabric.sh" - - -def run_installer(tmp_path: Path, *, present: bool = True, **overrides: str): - efa = tmp_path / "efa" - if present: - for filename in ("include/rdma/fabric.h", "lib/libfabric.so"): - path = efa / filename - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("original") - binary = efa / "bin/fi_info" - binary.parent.mkdir() - binary.write_text("#!/bin/sh\nprintf 'efa:\\nshm:\\n'\n") - binary.chmod(0o755) - installer = tmp_path / "install_libfabric.sh" - installer.write_text( - INSTALLER.read_text() - .replace("/opt/amazon/efa", str(efa)) - .replace("/third-party-source", str(tmp_path / "source")) - ) - patch = tmp_path / "patches/libfabric-2.4.0amzn5.0-cuda-vmm.patch" - patch.parent.mkdir() - patch.touch() - command = r""" -pkg-config() { printf '%s\n' "$VERSION"; } -curl() { touch "$CALLED"; } -sha256sum() { cat >/dev/null; return "$CHECKSUM_RC"; } -patch() { return "$PATCH_RC"; } -nproc() { echo 1; } -tar() { - if [ "$1" = -xzf ]; then - mkdir libfabric-2.4.0amzn5.0 - printf '#!/bin/sh\nexit 0\n' > libfabric-2.4.0amzn5.0/autogen.sh - cp libfabric-2.4.0amzn5.0/autogen.sh libfabric-2.4.0amzn5.0/configure - chmod +x libfabric-2.4.0amzn5.0/{autogen.sh,configure} - for feature in HAVE_CUDA HAVE_CUDA_CTX_SYNC_MEMOPS HAVE_GDRCOPY HAVE_NEURON; do - [ "$feature" = "$MISSING_FEATURE" ] || echo "#define $feature 1" - done > libfabric-2.4.0amzn5.0/config.h - fi -} -make() { - if [[ "${1:-}" = DESTDIR=* ]]; then - target="${1#DESTDIR=}$EFA_TEST_PATH" - mkdir -p "$target/lib" "$target/bin" - echo patched > "$target/lib/libfabric.so" - printf '#!/bin/sh\nprintf "%s"\n' "$NEW_PROVIDERS" > "$target/bin/fi_info" - chmod +x "$target/bin/fi_info" - fi -} -ldconfig() { touch "$INSTALLED"; } -source "$INSTALLER" -""" - env = { - "PATH": os.environ["PATH"], - "INSTALLER": str(installer), - "EFA_TEST_PATH": str(efa), - "CALLED": str(tmp_path / "download-started"), - "INSTALLED": str(tmp_path / "installed"), - "VERSION": "2.4.0amzn5.0", - "CHECKSUM_RC": "0", - "PATCH_RC": "0", - "MISSING_FEATURE": "", - "NEW_PROVIDERS": "shm:\\nefa:\\n", - **overrides, - } - return subprocess.run( - ["bash", "-c", command], env=env, capture_output=True, text=True, timeout=30 - ) - - -def test_no_bundled_efa_skips_download(tmp_path: Path) -> None: - result = run_installer(tmp_path, present=False) - assert result.returncode == 0, result.stderr - assert not (tmp_path / "download-started").exists() - - -def test_version_change_requires_revalidation(tmp_path: Path) -> None: - result = run_installer(tmp_path, VERSION="2.7.0amzn1.0") - assert result.returncode != 0 - assert "Revalidate" in result.stderr - assert not (tmp_path / "download-started").exists() - - -@pytest.mark.parametrize( - "failure", - [ - {"CHECKSUM_RC": "1"}, - {"PATCH_RC": "1"}, - {"MISSING_FEATURE": "HAVE_NEURON"}, - {"NEW_PROVIDERS": "efa:\\n"}, - ], -) -def test_failed_validation_preserves_original_library( - tmp_path: Path, failure: dict[str, str] -) -> None: - result = run_installer(tmp_path, **failure) - assert result.returncode != 0 - assert not (tmp_path / "installed").exists() - assert (tmp_path / "efa/lib/libfabric.so").read_text() == "original" - - -def test_matching_providers_allow_installation(tmp_path: Path) -> None: - result = run_installer(tmp_path) - assert result.returncode == 0, result.stdout + result.stderr - assert (tmp_path / "installed").exists() - assert (tmp_path / "efa/lib/libfabric.so").read_text().strip() == "patched" From 0468906c414737f7184d29c526c7d63518fa53e7 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Thu, 15 Oct 2026 08:57:43 +0800 Subject: [PATCH 14/27] Revert "Fix NIXL LIBFABRIC context selection for CUDA VMM" This reverts commit 9302e61f61e6f53df0d8407923b274f748137e29. Original pre-rebase commit: 4f5be3f6d3083dd9ecb1208a976b9f91860cd260. Keep this PR focused on building the LIBFABRIC plugin with bundled dependencies. Track the CUDA VMM context fix upstream in ai-dynamo/nixl#2425 rather than carrying a downstream patch in the container build. Signed-off-by: Yanchao Lu --- docker/common/install_nixl.sh | 6 +---- .../nixl-v1.4.0-cuda-vmm-context.patch | 26 ------------------- tests/unittest/scripts/test_install_nixl.py | 12 +-------- 3 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index 591667c4c8d0..c96fa1f38fb6 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -1,11 +1,8 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 set -ex -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Authenticate the github.com clone below; no-op when no token is available. -source "${SCRIPT_DIR}/github_auth.sh" +source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh" GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" @@ -43,7 +40,6 @@ pip3 install meson ninja pybind11 setuptools git clone --depth 1 -b ${NIXL_VERSION} ${NIXL_REPO} cd nixl -git apply "${SCRIPT_DIR}/patches/nixl-v1.4.0-cuda-vmm-context.patch" CUDA_SO_PATH=$(find "/usr/local" -name "libcuda.so.1" 2>/dev/null | head -n1) diff --git a/docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch b/docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch deleted file mode 100644 index 7995e0d97477..000000000000 --- a/docker/common/patches/nixl-v1.4.0-cuda-vmm-context.patch +++ /dev/null @@ -1,26 +0,0 @@ -SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Subject: Fall back to the device primary context for CUDA VMM allocations -Upstream: https://github.com/ai-dynamo/nixl/pull/2425 - ---- a/src/plugins/libfabric/libfabric_backend.cpp -+++ b/src/plugins/libfabric/libfabric_backend.cpp -@@ -267,6 +267,8 @@ - if (ret) return ret; - if (!is_dev) return 0; - if (dev != expected_dev) return -1; -+ // Context-independent allocations need the per-device primary-context path. -+ if (ctx == nullptr) return -1; - - if (pthrCudaCtx_) { - if (pthrCudaCtx_ != ctx) return -1; -@@ -900,7 +902,7 @@ - if (use_cuda_addr_wa) { - bool need_restart; - if (vramUpdateCtx((void *)mem.addr, mem.devId, need_restart)) { -- NIXL_INFO << "Multi-GPU detected (device " << mem.devId -+ NIXL_INFO << "CUDA pointer context unavailable or incompatible (device " << mem.devId - << "), using cudaSetDevice fallback"; - { - const std::lock_guard lock(cuda_ctx_mutex_); diff --git a/tests/unittest/scripts/test_install_nixl.py b/tests/unittest/scripts/test_install_nixl.py index 39962f76e1c3..97d6a4a13b9d 100644 --- a/tests/unittest/scripts/test_install_nixl.py +++ b/tests/unittest/scripts/test_install_nixl.py @@ -54,13 +54,7 @@ def test_bundled_dependencies_require_development_files( meson_args = tmp_path / "meson_args" command = r""" pip3() { :; } -git() { - case "$1" in - clone) mkdir nixl ;; - apply) printf '%s\n' "$2" > "$NIXL_PATCH" ;; - *) return 1 ;; - esac -} +git() { mkdir nixl; } find() { printf '%s\n' /usr/local/cuda/lib64/libcuda.so.1; } meson() { printf '%s\n' "$@" > "$MESON_ARGS" @@ -74,7 +68,6 @@ def test_bundled_dependencies_require_development_files( "INSTALL_NIXL": str(installer), "MESON_ARGS": str(meson_args), "MESON_PKG_CONFIG_PATH": str(tmp_path / "meson_pkg_config_path"), - "NIXL_PATCH": str(tmp_path / "nixl_patch"), } if pkg_config_path is not None: env["PKG_CONFIG_PATH"] = pkg_config_path @@ -88,9 +81,6 @@ def test_bundled_dependencies_require_development_files( ) assert result.returncode == 0, result.stderr - assert (tmp_path / "nixl_patch").read_text().strip() == str( - tmp_path / "patches/nixl-v1.4.0-cuda-vmm-context.patch" - ) args = meson_args.read_text().splitlines() expected_path = str(efa_path) if with_header and with_library else "" assert [arg for arg in args if arg.startswith("-Dlibfabric_path=")] == [ From 8c5b54cd81e55020c567a72198354cf2b705e08e Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Thu, 15 Oct 2026 08:58:45 +0800 Subject: [PATCH 15/27] Document bundled EFA limits without downstream dependency patches Keep the plugin directory variable correction and installer license headers. Describe the observed KV cache V2 limitation and upstream dependency requirements without changing the default backend or KV cache manager. Signed-off-by: Yanchao Lu --- docker/common/install_base.sh | 2 ++ docker/common/install_nixl.sh | 2 ++ examples/disaggregated/README.md | 17 ++++++++++------- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docker/common/install_base.sh b/docker/common/install_base.sh index 20f9060f9e56..690123d867be 100644 --- a/docker/common/install_base.sh +++ b/docker/common/install_base.sh @@ -1,4 +1,6 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 set -ex diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index c96fa1f38fb6..e37dee93aeba 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -1,4 +1,6 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 set -ex # Authenticate the github.com clone below; no-op when no token is available. diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index 6cdaceac83e6..8cdb4b4fb095 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -42,7 +42,9 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX ### LIBFABRIC Backend Setup -**Note:** Containers built from NGC PyTorch base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). With these containers on EFA-enabled instances, no extra library installation is needed — set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC`. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. +**Note:** Containers built from NGC PyTorch base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select this backend. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. + +**KV cache V2 limitation:** A two-node B300 test using NGC PyTorch 26.09, NIXL 1.4.0 and AWS libfabric 2.4.0amzn5.0 failed on the LIBFABRIC/EFA path with CUDA VMM allocations. The failures involve [NIXL allocation-context handling](https://github.com/ai-dynamo/nixl/pull/2425) and [libfabric CUDA synchronization](https://github.com/ofiwg/libfabric/issues/12853). The container build does not patch either dependency; this path needs fixes delivered through NIXL and the official EFA Installer. Plugin availability alone does not establish V2 compatibility. KV cache V1 passed the Qwen3-0.6B TP1/PP1 control, which does not establish support for other configurations. UCX remains the default backend. #### Prerequisites @@ -51,7 +53,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX **Required Dependencies:** **Libfabric** -- Bundled in NGC PyTorch base images with AWS EFA at `/opt/amazon/efa`. No extra library installation is needed when using these containers on EFA-enabled instances. +- Reuse the AWS EFA installation bundled in NGC PyTorch base images at `/opt/amazon/efa`, subject to the KV cache V2 limitation above. - For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) - **Minimum required version:** v1.21.0 - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) @@ -62,7 +64,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX - For other base images or custom MPI installations, provide compatible hwloc development files and ensure their `hwloc.pc` is discoverable through `PKG_CONFIG_PATH`. **Network Hardware Requirements:** -- Validated compatibility with AWS EFA (Elastic Fabric Adapter) +- AWS EFA (Elastic Fabric Adapter), with compatible KV cache settings and dependency versions as described above ##### For UCX Backend @@ -93,7 +95,7 @@ If you have a pre-compiled `libplugin_LIBFABRIC.so` that matches your NIXL versi 1. Place the plugin file in a directory of your choice 2. Set the environment variable to point to the plugin directory: ```bash - export NIXL_PLUGINS_DIR=/path/to/plugin/directory + export NIXL_PLUGIN_DIR=/path/to/plugin/directory export TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC ``` 3. Ensure the plugin was built with the same NIXL version as in your container @@ -147,7 +149,7 @@ CUDA_VISIBLE_DEVICES=2 trtllm-serve TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ export TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC # If using pre-compiled plugin: -# export NIXL_PLUGINS_DIR=/path/to/plugin/directory +# export NIXL_PLUGIN_DIR=/path/to/plugin/directory # For AWS EFA (optional): # export FI_PROVIDER=efa @@ -184,7 +186,7 @@ CUDA_VISIBLE_DEVICES=2 trtllm-serve TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ - Standard UCX environment variables apply **For LIBFABRIC backend:** -- `NIXL_PLUGINS_DIR`: Directory containing the NIXL LIBFABRIC plugin (`libplugin_LIBFABRIC.so`) if using pre-compiled plugin +- `NIXL_PLUGIN_DIR`: Directory containing the NIXL LIBFABRIC plugin (`libplugin_LIBFABRIC.so`) if using pre-compiled plugin - `FI_PROVIDER`: Specifies the libfabric provider to use (e.g., `efa` for AWS EFA) - `FI_EFA_USE_DEVICE_RDMA`: Set to `1` to enable GPU Direct RDMA on AWS EFA (if supported) - `FI_LOG_LEVEL`: Controls libfabric logging verbosity (e.g., `warn`, `info`, `debug`) @@ -602,4 +604,5 @@ A: Check that the container includes the NIXL LIBFABRIC plugin and the dependenc 1. **Missing plugin**: Older or custom images without libfabric development files at build time may not include the plugin. Follow [LIBFABRIC Backend Setup](#libfabric-backend-setup) to install it. 2. **EFA devices unavailable**: If using EFA, confirm the instance supports it and the devices are passed through to the container. 3. **Custom libfabric version mismatch**: If you rebuilt NIXL against a different libfabric installation, confirm it meets the minimum required version (v1.21.0) and that the plugin was built against that same libfabric. -4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGINS_DIR` to point to its directory. +4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGIN_DIR` to point to its directory. +5. **KV cache V2 with CUDA VMM**: A plugin can load successfully while transfers fail because of dependency limitations. See [LIBFABRIC Backend Setup](#libfabric-backend-setup) for the affected versions and upstream fixes. From e0b4d06ff81a5ab41b99dc72574ac1a4a7633243 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Thu, 15 Oct 2026 08:59:05 +0800 Subject: [PATCH 16/27] Restore main CI images pending an unpatched staging rebuild Retain both historical staging-image commits. Replace their active image references with the main baseline because the latest staging images contain the reverted NIXL and libfabric patches. Fresh staging tags will be selected in a subsequent commit after rebuilding this revision. Signed-off-by: Yanchao Lu --- jenkins/current_image_tags.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index ec60c2c87b7d..428f962e04f1 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm -LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-tritondevel-torch_skip-27399f8-github-pr-19783-64158 -LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-tritondevel-torch_skip-27399f8-github-pr-19783-64158 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py310-27399f8-github-pr-19783-64158 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:x86_64-rockylinux8-torch_skip-py312-27399f8-github-pr-19783-64158 -LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm-staging/tritondevel:sbsa-ubuntu24-torch_skip-py312-27399f8-github-pr-19783-64158 +LLM_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.09-py3-x86_64-ubuntu24.04-skip-tritondevel-202610130652-20175 +LLM_SBSA_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-26.09-py3-sbsa-ubuntu24.04-skip-tritondevel-202610130652-20175 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.4.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202610130652-20175 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.4.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202610130652-20175 +LLM_SBSA_WHEEL_DOCKER_IMAGE=artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:cuda-13.4.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202610130652-20175 From 85f73b1c5df37dce53e9354d91fd55c7b8a2115b Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Thu, 15 Oct 2026 15:19:50 +0800 Subject: [PATCH 17/27] Clarify KV cache V2 EFA findings and NIXL backend terminology Signed-off-by: Yanchao Lu --- examples/disaggregated/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index 8cdb4b4fb095..5bd432d22c8a 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -32,19 +32,19 @@ Keep `cache_transceiver_config` identical in the two files. For the remaining fi ## NIXL Backend Configuration -NIXL supports multiple underlying communication backends for KV cache exchange. The backend can be configured using the `TRTLLM_NIXL_KVCACHE_BACKEND` environment variable. +With `cache_transceiver_config.backend: NIXL`, the `TRTLLM_NIXL_KVCACHE_BACKEND` environment variable selects the underlying NIXL backend for KV cache exchange. The NIXL UCX backend is distinct from TRT-LLM's direct UCX backend, selected with `cache_transceiver_config.backend: UCX`. **Supported NIXL backends:** - **UCX** (default) - **LIBFABRIC** (available from v0.16.0) -If an unsupported backend is specified, NIXL will automatically fall back to UCX. +If an unsupported NIXL backend is specified, TRT-LLM selects the NIXL UCX backend. ### LIBFABRIC Backend Setup **Note:** Containers built from NGC PyTorch base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select this backend. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. -**KV cache V2 limitation:** A two-node B300 test using NGC PyTorch 26.09, NIXL 1.4.0 and AWS libfabric 2.4.0amzn5.0 failed on the LIBFABRIC/EFA path with CUDA VMM allocations. The failures involve [NIXL allocation-context handling](https://github.com/ai-dynamo/nixl/pull/2425) and [libfabric CUDA synchronization](https://github.com/ofiwg/libfabric/issues/12853). The container build does not patch either dependency; this path needs fixes delivered through NIXL and the official EFA Installer. Plugin availability alone does not establish V2 compatibility. KV cache V1 passed the Qwen3-0.6B TP1/PP1 control, which does not establish support for other configurations. UCX remains the default backend. +**KV cache V2 limitation with the NIXL LIBFABRIC backend:** Two-node B300 testing with NGC PyTorch 26.09, NIXL 1.4.0 and AWS libfabric 2.4.0amzn5.0 exposed [NIXL allocation-context handling](https://github.com/ai-dynamo/nixl/pull/2425) and [libfabric CUDA VMM synchronization](https://github.com/ofiwg/libfabric/issues/12853) failures on the EFA RDM path. A diagnostic backport of the libfabric fixes alone still failed NIXL's context check. With that backport and unmodified NIXL, `NIXL_DISABLE_CUDA_ADDR_WA=1` allowed KV cache V2 inference to complete; this workaround does not fix the bundled libfabric. The container patches neither dependency and does not set this environment variable by default. Plugin availability does not establish KV cache V2 compatibility over EFA. KV cache V1 passed the Qwen3-0.6B TP1/PP1 control only. The NIXL UCX backend remains the default. #### Prerequisites @@ -66,7 +66,7 @@ If an unsupported backend is specified, NIXL will automatically fall back to UCX **Network Hardware Requirements:** - AWS EFA (Elastic Fabric Adapter), with compatible KV cache settings and dependency versions as described above -##### For UCX Backend +##### For the NIXL UCX Backend UCX is typically pre-installed in NVIDIA GPU containers. No additional installation is usually required. From bfd443f3fa2b5ccac4810972cfeae787a38ae7a3 Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Fri, 23 Oct 2026 19:22:45 +0000 Subject: [PATCH 18/27] Add EFA Installer 1.51 Signed-off-by: Erez Zarum --- docker/Dockerfile.multi | 1 + docker/common/install_efa_installer.sh | 33 ++++++++++++++++++++++++++ docker/common/install_nixl.sh | 33 ++++++++++++++++++++------ 3 files changed, 60 insertions(+), 7 deletions(-) create mode 100755 docker/common/install_efa_installer.sh diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index c6712f3c8b4e..b8666137ae53 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -96,6 +96,7 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ --mount=type=secret,id=github_clone_token \ . /opt/docker/common/github_auth.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_ucx.sh && \ + bash /opt/docker/common/install_efa_installer.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_nixl.sh && \ bash /opt/docker/common/install_etcd.sh && \ rm -rf /root/.cache/uv/archive-v0 && \ diff --git a/docker/common/install_efa_installer.sh b/docker/common/install_efa_installer.sh new file mode 100755 index 000000000000..ea98db3501da --- /dev/null +++ b/docker/common/install_efa_installer.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -ex + +# Update EFA software stack only on Ubuntu images; skip elsewhere. +OS=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [ "${OS}" != "ubuntu" ]; then + echo "Skipping EFA installer: OS '${OS}'." + exit 0 +fi + +EFA_INSTALLER_VERSION="${EFA_INSTALLER_VERSION:-1.51.0}" +EFA_INSTALLER_URL="https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" + +WORK_DIR="$(mktemp -d)" +cd "${WORK_DIR}" + +curl --retry 3 --retry-delay 2 -fsSL -o "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \ + "${EFA_INSTALLER_URL}" +tar -xf "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" +cd aws-efa-installer + +apt-get update + +# --skip-mpi: NGC images already ship HPC-X / Open MPI; do not overwrite it. +# --skip-rdma-core: use the base image's rdma-core. +# --build-ngc: build against the NGC-provided userspace rather than installing +# the bundled one. +./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc + +cd ../../ +rm -rf "${WORK_DIR}" diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index e37dee93aeba..a8264e010499 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -3,17 +3,35 @@ # SPDX-License-Identifier: Apache-2.0 set -ex +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Authenticate the github.com clone below; no-op when no token is available. -source "$(dirname "${BASH_SOURCE[0]}")/github_auth.sh" +source "${SCRIPT_DIR}/github_auth.sh" GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" -# Default prefix for AWS EFA bundled with NGC PyTorch images. -LIBFABRIC_INSTALL_PATH="/opt/amazon/efa" -if [[ ! -f "${LIBFABRIC_INSTALL_PATH}/include/rdma/fabric.h" || - ! -f "${LIBFABRIC_INSTALL_PATH}/lib/libfabric.so" ]]; then - LIBFABRIC_INSTALL_PATH="" -fi + +# Interm build, minimum supported libfabric version the NIXL LIBFABRIC plugin works with, retained for backward compatibility. +LIBFABRIC_VERSION="v1.21.0" +LIBFABRIC_INSTALL_PATH="/tmp/libfabric" +wget "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 +tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 +cd libfabric-* +./autogen.sh +./configure --prefix=${LIBFABRIC_INSTALL_PATH} \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --enable-efa \ + --with-cuda=/usr/local/cuda \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen +make -j${NPROC:-$(nproc)} +make install +cd ../ +rm -rf libfabric* + # NGC PyTorch's default Open MPI comes from HPC-X and is exposed at /usr/local/mpi. # Reuse the hwloc bundled with that MPI installation for the EFA plugin. HWLOC_INSTALL_PATH="/usr/local/mpi" @@ -69,6 +87,7 @@ meson setup builddir \ cd builddir && ninja install cd ../.. rm -rf nixl* # Remove NIXL source tree to save space +rm -rf /tmp/libfabric # Remove libfabric interim build export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH # Consumers import `nixl`, but the build above installs the backend as From 2ff79202e70c963a5f751b113c47b16b2053d15c Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Thu, 29 Oct 2026 20:44:25 +0000 Subject: [PATCH 19/27] Bump NIXL to v1.5.2 Signed-off-by: Erez Zarum --- docker/common/install_nixl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index 620e7aa9e919..f116be1171f0 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -42,7 +42,7 @@ if [[ -n "${LIBFABRIC_INSTALL_PATH}" && export PKG_CONFIG_PATH="${HWLOC_INSTALL_PATH}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" fi CUDA_PATH="/usr/local/cuda" -NIXL_VERSION="v1.5.0" +NIXL_VERSION="v1.5.2" NIXL_REPO="https://github.com/ai-dynamo/nixl.git" OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH From dd86d273698ffd4c12a7331c6a9b751fc3ec9b81 Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Thu, 29 Oct 2026 21:37:03 +0000 Subject: [PATCH 20/27] Update disagg README Signed-off-by: Erez Zarum --- examples/disaggregated/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index 422d9fb3ed37..aac1dd93905a 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -52,9 +52,7 @@ If an unsupported NIXL backend is specified, TRT-LLM selects the NIXL UCX backen ### LIBFABRIC Backend Setup -**Note:** Containers built from NGC PyTorch base images with AWS EFA development files at `/opt/amazon/efa` build the NIXL-LIBFABRIC plugin against that installation by default (see `docker/common/install_nixl.sh`). Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select this backend. Other base images fall back to system libfabric detection and may not include the plugin. Use the installation options below if the plugin is missing or you need a custom libfabric installation. - -**KV cache V2 limitation with the NIXL LIBFABRIC backend:** Two-node B300 testing with NGC PyTorch 26.09, NIXL 1.4.0 and AWS libfabric 2.4.0amzn5.0 exposed [NIXL allocation-context handling](https://github.com/ai-dynamo/nixl/pull/2425) and [libfabric CUDA VMM synchronization](https://github.com/ofiwg/libfabric/issues/12853) failures on the EFA RDM path. A diagnostic backport of the libfabric fixes alone still failed NIXL's context check. With that backport and unmodified NIXL, `NIXL_DISABLE_CUDA_ADDR_WA=1` allowed KV cache V2 inference to complete; this workaround does not fix the bundled libfabric. The container patches neither dependency and does not set this environment variable by default. Plugin availability does not establish KV cache V2 compatibility over EFA. KV cache V1 passed the Qwen3-0.6B TP1/PP1 control only. The NIXL UCX backend remains the default. +**Note:** The NIXL-LIBFABRIC plugin is always built against libfabric 1.21.0, the same version used by NIXL upstream, to keep backward compatibility (see `docker/common/install_nixl.sh`). At runtime, containers built from NGC PyTorch base images use the AWS EFA installation at `/opt/amazon/efa`. Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select this backend. Other base images fall back to system libfabric detection if available. Use the installation options below if the plugin is missing or you need a custom libfabric installation. #### Prerequisites @@ -63,7 +61,7 @@ If an unsupported NIXL backend is specified, TRT-LLM selects the NIXL UCX backen **Required Dependencies:** **Libfabric** -- Reuse the AWS EFA installation bundled in NGC PyTorch base images at `/opt/amazon/efa`, subject to the KV cache V2 limitation above. +- Reuse the AWS EFA installation bundled in NGC PyTorch base images at `/opt/amazon/efa`. - For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) - **Minimum required version:** v1.21.0 - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) @@ -94,7 +92,7 @@ Skip these steps if your container already includes the NIXL LIBFABRIC plugin fo 3. **Reinstall NIXL after installing libfabric:** - After installing libfabric and hwloc, you must rebuild NIXL to generate the LIBFABRIC plugin against your custom path - - You can base your installation on the TensorRT LLM NIXL installation script located at `docker/common/install_nixl.sh`, which selects the bundled AWS EFA path when its header and link library are present, or leaves `-Dlibfabric_path` empty for system detection + - You can base your installation on the TensorRT LLM NIXL installation script located at `docker/common/install_nixl.sh`, which uses libfabric v1.21.0 or leaves `-Dlibfabric_path` empty for system detection - For a custom installation, set `LIBFABRIC_INSTALL_PATH=/path/to/libfabric` immediately before the script's `meson setup` command, keeping the other Meson options unchanged - For more details, see the [NIXL LIBFABRIC Plugin documentation](https://github.com/ai-dynamo/nixl/tree/6ee64753605b3110f8ef96c7cfc2f1315675c9c7/src/plugins/libfabric#nixl-libfabric-plugin) @@ -409,12 +407,12 @@ Skip this step if the container already includes AWS EFA at `/opt/amazon/efa`. O # Install AWS EFA library (required for LIBFABRIC with EFA) curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz tar -xf aws-efa-installer-latest.tar.gz -cd aws-efa-installer && ./efa_installer.sh --yes --skip-kmod +cd aws-efa-installer && ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify ``` ##### 3. Rebuild NIXL with EFA Support if Needed -Skip this step if the NIXL LIBFABRIC plugin was already built against the container's EFA installation. If you installed or replaced EFA, follow the rebuild instructions in [LIBFABRIC Backend Setup](#libfabric-backend-setup), using `/opt/amazon/efa` as the libfabric path. +Skip this step if the NIXL LIBFABRIC plugin was already built against the container's libfabric version. ##### 4. Configure and Launch Services From 33056dfd3f3736bfecd5531243fd819df675404b Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sat, 31 Oct 2026 23:48:59 +0800 Subject: [PATCH 21/27] Align EFA and NIXL installers, dependencies and regression tests Signed-off-by: Yanchao Lu --- docker/common/install_efa_installer.sh | 14 +- docker/common/install_nixl.sh | 28 +-- examples/disaggregated/README.md | 30 ++-- requirements-dev.txt | 2 +- .../scripts/test_install_efa_installer.py | 80 +++++++++ tests/unittest/scripts/test_install_nixl.py | 167 ++++++++++++------ 6 files changed, 232 insertions(+), 89 deletions(-) create mode 100644 tests/unittest/scripts/test_install_efa_installer.py diff --git a/docker/common/install_efa_installer.sh b/docker/common/install_efa_installer.sh index ea98db3501da..0fdd241c2c86 100755 --- a/docker/common/install_efa_installer.sh +++ b/docker/common/install_efa_installer.sh @@ -4,9 +4,9 @@ set -ex # Update EFA software stack only on Ubuntu images; skip elsewhere. -OS=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') -if [ "${OS}" != "ubuntu" ]; then - echo "Skipping EFA installer: OS '${OS}'." +. /etc/os-release +if [ "${ID}" != "ubuntu" ]; then + echo "Skipping EFA installer: OS '${ID}'." exit 0 fi @@ -14,6 +14,7 @@ EFA_INSTALLER_VERSION="${EFA_INSTALLER_VERSION:-1.51.0}" EFA_INSTALLER_URL="https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" WORK_DIR="$(mktemp -d)" +trap 'rm -rf "${WORK_DIR}"' EXIT cd "${WORK_DIR}" curl --retry 3 --retry-delay 2 -fsSL -o "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \ @@ -25,9 +26,6 @@ apt-get update # --skip-mpi: NGC images already ship HPC-X / Open MPI; do not overwrite it. # --skip-rdma-core: use the base image's rdma-core. -# --build-ngc: build against the NGC-provided userspace rather than installing -# the bundled one. +# --build-ngc: install the EFA userspace packages without host driver installation, +# device verification, limit configuration or dependency replacement. ./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc - -cd ../../ -rm -rf "${WORK_DIR}" diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index f116be1171f0..feb60b0db8a2 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -10,33 +10,38 @@ source "${SCRIPT_DIR}/github_auth.sh" GITHUB_URL="https://github.com" UCX_INSTALL_PATH="/usr/local/ucx/" -# Interm build, minimum supported libfabric version the NIXL LIBFABRIC plugin works with, retained for backward compatibility. +# Build against the oldest supported libfabric ABI, independently of the runtime EFA installation. +# Keep the sources and build-only libraries isolated from the caller's working directory. +WORK_DIR=$(mktemp -d) +trap 'rm -rf "${WORK_DIR}"' EXIT +cd "${WORK_DIR}" LIBFABRIC_VERSION="v1.21.0" -LIBFABRIC_INSTALL_PATH="/tmp/libfabric" -wget "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 +LIBFABRIC_INSTALL_PATH="${WORK_DIR}/libfabric" +wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 -cd libfabric-* +cd "libfabric-${LIBFABRIC_VERSION#v}" ./autogen.sh -./configure --prefix=${LIBFABRIC_INSTALL_PATH} \ +./configure --prefix="${LIBFABRIC_INSTALL_PATH}" \ --disable-verbs \ --disable-psm3 \ --disable-opx \ --disable-usnic \ + --disable-rstream \ --enable-efa \ --with-cuda=/usr/local/cuda \ --enable-cuda-dlopen \ --with-gdrcopy \ --enable-gdrcopy-dlopen -make -j${NPROC:-$(nproc)} +make -j"${NPROC:-$(nproc)}" make install cd ../ -rm -rf libfabric* +rm -rf "libfabric-${LIBFABRIC_VERSION#v}" # NGC PyTorch's default Open MPI comes from HPC-X and is exposed at /usr/local/mpi. # Reuse the hwloc bundled with that MPI installation for the EFA plugin. HWLOC_INSTALL_PATH="/usr/local/mpi" -if [[ -n "${LIBFABRIC_INSTALL_PATH}" && - -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" && +if [[ -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" && -f "${HWLOC_INSTALL_PATH}/lib/libhwloc.so" && -f "${HWLOC_INSTALL_PATH}/lib/pkgconfig/hwloc.pc" ]]; then export PKG_CONFIG_PATH="${HWLOC_INSTALL_PATH}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" @@ -71,7 +76,7 @@ fi CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH -# An empty libfabric_path uses optional system detection; no match skips the plugin. +# This build-only prefix is removed on exit; runtime libfabric comes from the image or host. meson setup builddir \ -Ducx_path=$UCX_INSTALL_PATH \ -Dlibfabric_path="${LIBFABRIC_INSTALL_PATH}" \ @@ -86,8 +91,7 @@ meson setup builddir \ cd builddir && ninja install cd ../.. -rm -rf nixl* # Remove NIXL source tree to save space -rm -rf /tmp/libfabric # Remove libfabric interim build +rm -rf nixl # Remove NIXL source tree to save space export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH # Consumers import `nixl`, but the build above installs the backend as diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index aac1dd93905a..a7f851e59197 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -52,7 +52,9 @@ If an unsupported NIXL backend is specified, TRT-LLM selects the NIXL UCX backen ### LIBFABRIC Backend Setup -**Note:** The NIXL-LIBFABRIC plugin is always built against libfabric 1.21.0, the same version used by NIXL upstream, to keep backward compatibility (see `docker/common/install_nixl.sh`). At runtime, containers built from NGC PyTorch base images use the AWS EFA installation at `/opt/amazon/efa`. Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select this backend. Other base images fall back to system libfabric detection if available. Use the installation options below if the plugin is missing or you need a custom libfabric installation. +**Container builds:** `docker/common/install_nixl.sh` builds NIXL 1.5.2 and its LIBFABRIC plugin against a temporary libfabric 1.21.0 installation, matching the NIXL upstream wheel's ABI baseline. That installation is removed after the build. Ubuntu builds through `docker/Dockerfile.multi`, including NGC PyTorch images, install the official EFA Installer 1.51.0 userspace libraries at `/opt/amazon/efa` while preserving the existing MPI and rdma-core. Other base images must provide a compatible runtime libfabric and the appropriate provider. Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select the NIXL LIBFABRIC backend; the NIXL UCX backend remains the default. + +**KV cache V2 over EFA:** Use NIXL 1.5.2 and EFA Installer 1.51.0 or later. These releases include the CUDA VMM context and synchronization fixes, plus EFA completion fixes. The libfabric 1.21.0 build baseline is an ABI compatibility choice, not a guarantee that older runtime libraries support CUDA VMM. `NIXL_DISABLE_CUDA_ADDR_WA=1` is not required for the context-independent allocation fix in NIXL 1.5.2. See the [NIXL release notes](https://github.com/ai-dynamo/nixl/releases/tag/v1.5.2) and [libfabric CUDA VMM fix](https://github.com/ofiwg/libfabric/pull/12918). #### Prerequisites @@ -61,13 +63,13 @@ If an unsupported NIXL backend is specified, TRT-LLM selects the NIXL UCX backen **Required Dependencies:** **Libfabric** -- Reuse the AWS EFA installation bundled in NGC PyTorch base images at `/opt/amazon/efa`. +- On AWS, use the EFA installation at `/opt/amazon/efa`, updated by the container build as described above. - For a custom libfabric installation, see [https://ofiwg.github.io/libfabric/](https://ofiwg.github.io/libfabric/) -- **Minimum required version:** v1.21.0 +- **Minimum ABI baseline:** v1.21.0; the required runtime version also depends on the provider and memory type. - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) **hwloc** -- NGC PyTorch container builds reuse the hwloc development files bundled with HPC-X under `/usr/local/mpi`. When the bundled EFA and hwloc development files are present, `docker/common/install_nixl.sh` prepends `/usr/local/mpi/lib/pkgconfig` to `PKG_CONFIG_PATH`; no extra `libhwloc-dev` installation is needed for this configuration. +- NGC PyTorch container builds reuse the hwloc development files bundled with HPC-X under `/usr/local/mpi`. When those hwloc development files are present, `docker/common/install_nixl.sh` prepends `/usr/local/mpi/lib/pkgconfig` to `PKG_CONFIG_PATH`; no extra `libhwloc-dev` installation is needed for this configuration. - hwloc is used to understand the underlying architecture to optimize application performance - For other base images or custom MPI installations, provide compatible hwloc development files and ensure their `hwloc.pc` is discoverable through `PKG_CONFIG_PATH`. @@ -92,7 +94,7 @@ Skip these steps if your container already includes the NIXL LIBFABRIC plugin fo 3. **Reinstall NIXL after installing libfabric:** - After installing libfabric and hwloc, you must rebuild NIXL to generate the LIBFABRIC plugin against your custom path - - You can base your installation on the TensorRT LLM NIXL installation script located at `docker/common/install_nixl.sh`, which uses libfabric v1.21.0 or leaves `-Dlibfabric_path` empty for system detection + - You can base your installation on `docker/common/install_nixl.sh`, which always builds a temporary libfabric 1.21.0 installation for the plugin - For a custom installation, set `LIBFABRIC_INSTALL_PATH=/path/to/libfabric` immediately before the script's `meson setup` command, keeping the other Meson options unchanged - For more details, see the [NIXL LIBFABRIC Plugin documentation](https://github.com/ai-dynamo/nixl/tree/6ee64753605b3110f8ef96c7cfc2f1315675c9c7/src/plugins/libfabric#nixl-libfabric-plugin) @@ -399,20 +401,20 @@ resources: vpc.amazonaws.com/efa: 4 # Number of EFA network interfaces ``` -##### 2. Install EFA Libraries if Missing +##### 2. Install or Update EFA Libraries -Skip this step if the container already includes AWS EFA at `/opt/amazon/efa`. Otherwise, install the EFA libraries before rebuilding NIXL: +Skip this step if the container already includes EFA Installer 1.51.0 or later. Otherwise, update the EFA userspace libraries. For Ubuntu container builds with MPI and rdma-core already installed: ```bash -# Install AWS EFA library (required for LIBFABRIC with EFA) -curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz -tar -xf aws-efa-installer-latest.tar.gz -cd aws-efa-installer && ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify +# Update AWS EFA userspace without replacing MPI or rdma-core. +curl -fLO https://efa-installer.amazonaws.com/aws-efa-installer-1.51.0.tar.gz +tar -xf aws-efa-installer-1.51.0.tar.gz +cd aws-efa-installer && ./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc ``` ##### 3. Rebuild NIXL with EFA Support if Needed -Skip this step if the NIXL LIBFABRIC plugin was already built against the container's libfabric version. +Skip this step if the container already includes NIXL 1.5.2 or later with a LIBFABRIC plugin compatible with the runtime libfabric. The build-time and runtime libfabric versions do not need to be identical. ##### 4. Configure and Launch Services @@ -607,6 +609,6 @@ A: Check that the container includes the NIXL LIBFABRIC plugin and the dependenc 1. **Missing plugin**: Older or custom images without libfabric development files at build time may not include the plugin. Follow [LIBFABRIC Backend Setup](#libfabric-backend-setup) to install it. 2. **EFA devices unavailable**: If using EFA, confirm the instance supports it and the devices are passed through to the container. -3. **Custom libfabric version mismatch**: If you rebuilt NIXL against a different libfabric installation, confirm it meets the minimum required version (v1.21.0) and that the plugin was built against that same libfabric. +3. **Custom libfabric version mismatch**: Ensure the loaded runtime libfabric satisfies the plugin's ABI requirements. The container plugin is built against the 1.21.0 baseline; custom plugins built against newer libraries can require newer runtime symbols. 4. **Using a pre-compiled plugin**: If you have a compatible `libplugin_LIBFABRIC.so`, set `NIXL_PLUGIN_DIR` to point to its directory. -5. **KV cache V2 with CUDA VMM**: A plugin can load successfully while transfers fail because of dependency limitations. See [LIBFABRIC Backend Setup](#libfabric-backend-setup) for the affected versions and upstream fixes. +5. **KV cache V2 with CUDA VMM**: Plugin loading alone does not establish transfer compatibility. For EFA, check both the NIXL and EFA runtime versions against [LIBFABRIC Backend Setup](#libfabric-backend-setup), including the libraries actually loaded by the workers. diff --git a/requirements-dev.txt b/requirements-dev.txt index 42098a64b8f0..2e0a3471e318 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -70,7 +70,7 @@ opentelemetry-semantic-conventions-ai>=0.4.1 rapidfuzz==3.14.5 aiperf==0.8.0 nanobind>=2.9.0 -nixl-cu13==1.5.0 +nixl-cu13==1.5.2 cupti-python>=13.0,<13.5 nvidia-cuda-cupti>=13.0,<13.5 cxxfilt diff --git a/tests/unittest/scripts/test_install_efa_installer.py b/tests/unittest/scripts/test_install_efa_installer.py new file mode 100644 index 000000000000..3b412dedbc23 --- /dev/null +++ b/tests/unittest/scripts/test_install_efa_installer.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.cpu_only +INSTALL_EFA = Path(__file__).resolve().parents[3] / "docker/common/install_efa_installer.sh" + +_COMMAND = r""" +curl() { + printf '%s\n' "$@" > "$CURL_ARGS" + [ "$FAIL_STAGE" != download ] +} +tar() { + mkdir aws-efa-installer + cat > aws-efa-installer/efa_installer.sh <<'EOF' +#!/bin/bash +printf '%s\n' "$@" > "$EFA_ARGS" +[ "$FAIL_STAGE" != install ] +EOF + chmod +x aws-efa-installer/efa_installer.sh +} +apt-get() { printf '%s\n' "$@" > "$APT_ARGS"; } +source "$INSTALL_EFA" +""" + + +@pytest.mark.parametrize( + ("os_id", "version", "fail_stage"), + ( + ("rocky", None, ""), + ("ubuntu", None, ""), + ("ubuntu", "1.52.0", ""), + ("ubuntu", None, "download"), + ("ubuntu", None, "install"), + ), +) +def test_efa_container_install( + tmp_path: Path, os_id: str, version: str | None, fail_stage: str +) -> None: + os_release = tmp_path / "os-release" + os_release.write_text(f'ID="{os_id}"\n') + installer = tmp_path / "install_efa.sh" + installer.write_text(INSTALL_EFA.read_text().replace("/etc/os-release", str(os_release))) + scratch = tmp_path / "scratch" + scratch.mkdir() + env = { + "PATH": os.environ["PATH"], + "INSTALL_EFA": str(installer), + "TMPDIR": str(scratch), + "FAIL_STAGE": fail_stage, + "CURL_ARGS": str(tmp_path / "curl_args"), + "EFA_ARGS": str(tmp_path / "efa_args"), + "APT_ARGS": str(tmp_path / "apt_args"), + } + if version is not None: + env["EFA_INSTALLER_VERSION"] = version + result = subprocess.run( + ["bash", "-c", _COMMAND], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30 + ) + assert (result.returncode != 0) == bool(fail_stage), result.stderr + assert not list(scratch.iterdir()) + if os_id == "rocky": + assert not (tmp_path / "curl_args").exists() + assert not (tmp_path / "apt_args").exists() + elif not fail_stage: + assert (tmp_path / "efa_args").read_text().splitlines() == [ + "--skip-mpi", + "--skip-rdma-core", + "--build-ngc", + ] + assert (tmp_path / "apt_args").read_text().splitlines() == ["update"] + assert ( + f"https://efa-installer.amazonaws.com/aws-efa-installer-{version or '1.51.0'}.tar.gz" + in (tmp_path / "curl_args").read_text().splitlines() + ) diff --git a/tests/unittest/scripts/test_install_nixl.py b/tests/unittest/scripts/test_install_nixl.py index 97d6a4a13b9d..99cab4b6b5c6 100644 --- a/tests/unittest/scripts/test_install_nixl.py +++ b/tests/unittest/scripts/test_install_nixl.py @@ -9,70 +9,94 @@ pytestmark = pytest.mark.cpu_only -INSTALL_NIXL = Path(__file__).resolve().parents[3] / "docker/common/install_nixl.sh" +ROOT = Path(__file__).resolve().parents[3] +INSTALL_NIXL = ROOT / "docker/common/install_nixl.sh" +# Execute the full installer while replacing network, compilation and package installation. +_COMMAND = r""" +wget() { + [ "$FAIL_STAGE" != download ] || return 23 + : > libfabric.tar.bz2 +} +tar() { + mkdir libfabric-1.21.0 + printf '#!/bin/bash\nexit 0\n' > libfabric-1.21.0/autogen.sh + cat > libfabric-1.21.0/configure <<'EOF' +#!/bin/bash +[ "$FAIL_STAGE" != configure ] || exit 23 +printf '%s\n' "$@" > "$CONFIGURE_ARGS" +for arg in "$@"; do + case "$arg" in --prefix=*) mkdir -p "$(printf '%s' "$arg" | cut -c10-)" ;; esac +done +EOF + chmod +x libfabric-1.21.0/autogen.sh libfabric-1.21.0/configure +} +make() { :; } +nproc() { echo 2; } +pip3() { + printf '%s\n' "$@" >> "$PIP_ARGS" + if [ "$2" = "--no-deps" ] && [ "$FAIL_STAGE" = shim ]; then return 23; fi +} +git() { printf '%s\n' "$@" > "$GIT_ARGS"; mkdir nixl; } +find() { printf '%s\n' /usr/local/cuda/lib64/libcuda.so.1; } +uname() { echo "$TEST_ARCH"; } +meson() { + printf '%s\n' "$@" > "$MESON_ARGS" + if [ -n "$PKG_CONFIG_PATH" ]; then + printf '%s' "$PKG_CONFIG_PATH" > "$MESON_PKG_CONFIG_PATH" + else + printf '' > "$MESON_PKG_CONFIG_PATH" + fi + mkdir builddir +} +ninja() { [ "$FAIL_STAGE" != ninja ]; } +source "$INSTALL_NIXL" +printf '%s' "$LD_LIBRARY_PATH" > "$RESTORED_LD_PATH" +""" -@pytest.mark.parametrize("with_header", (False, True)) -@pytest.mark.parametrize("with_library", (False, True)) -@pytest.mark.parametrize( - "missing_hwloc", - (None, "all", "include/hwloc.h", "lib/libhwloc.so", "lib/pkgconfig/hwloc.pc"), -) -@pytest.mark.parametrize("pkg_config_path", (None, "/custom/lib/pkgconfig")) -def test_bundled_dependencies_require_development_files( - tmp_path: Path, - with_header: bool, - with_library: bool, - missing_hwloc: str | None, - pkg_config_path: str | None, -) -> None: - efa_path = tmp_path / "efa" - for present, name in ( - (with_header, "include/rdma/fabric.h"), - (with_library, "lib/libfabric.so"), - ): - if present: - dependency = efa_path / name - dependency.parent.mkdir(parents=True, exist_ok=True) - dependency.touch() +def _run_installer( + tmp_path: Path, + missing_hwloc: str | None = None, + pkg_config_path: str | None = None, + arch: str = "x86_64", + fail_stage: str = "", +) -> subprocess.CompletedProcess[str]: mpi_path = tmp_path / "mpi" for name in ("include/hwloc.h", "lib/libhwloc.so", "lib/pkgconfig/hwloc.pc"): if missing_hwloc not in ("all", name): dependency = mpi_path / name dependency.parent.mkdir(parents=True, exist_ok=True) dependency.touch() - - # Relocate NGC PyTorch's bundled EFA and MPI prefixes without root access. installer = tmp_path / "install_nixl.sh" - installer.write_text( - INSTALL_NIXL.read_text() - .replace("/opt/amazon/efa", str(efa_path)) - .replace("/usr/local/mpi", str(mpi_path)) - ) + installer.write_text(INSTALL_NIXL.read_text().replace("/usr/local/mpi", str(mpi_path))) (tmp_path / "github_auth.sh").write_text(":\n") - meson_args = tmp_path / "meson_args" - command = r""" -pip3() { :; } -git() { mkdir nixl; } -find() { printf '%s\n' /usr/local/cuda/lib64/libcuda.so.1; } -meson() { - printf '%s\n' "$@" > "$MESON_ARGS" - printf '%s' "${PKG_CONFIG_PATH-}" > "$MESON_PKG_CONFIG_PATH" - exit 0 -} -source "$INSTALL_NIXL" -""" + (tmp_path / "scratch").mkdir() + # These names used to be vulnerable to the installer's wildcard cleanup. + for name in ("libfabric-keep", "nixl-keep"): + (tmp_path / name).write_text("keep\n") env = { "PATH": os.environ["PATH"], "INSTALL_NIXL": str(installer), - "MESON_ARGS": str(meson_args), - "MESON_PKG_CONFIG_PATH": str(tmp_path / "meson_pkg_config_path"), + "TMPDIR": str(tmp_path / "scratch"), + "ENV": str(tmp_path / "shinit"), + "LD_LIBRARY_PATH": "/existing/lib", + "TEST_ARCH": arch, + "FAIL_STAGE": fail_stage, } + for name in ( + "CONFIGURE_ARGS", + "PIP_ARGS", + "GIT_ARGS", + "MESON_ARGS", + "MESON_PKG_CONFIG_PATH", + "RESTORED_LD_PATH", + ): + env[name] = str(tmp_path / name.lower()) if pkg_config_path is not None: env["PKG_CONFIG_PATH"] = pkg_config_path - result = subprocess.run( - ["bash", "-c", command], + return subprocess.run( + ["bash", "-c", _COMMAND], cwd=tmp_path, env=env, capture_output=True, @@ -80,15 +104,50 @@ def test_bundled_dependencies_require_development_files( timeout=30, ) + +def _assert_cleaned(tmp_path: Path) -> None: + assert not list((tmp_path / "scratch").iterdir()) + for name in ("libfabric-keep", "nixl-keep"): + assert (tmp_path / name).read_text() == "keep\n" + + +@pytest.mark.parametrize("arch", ("x86_64", "aarch64")) +@pytest.mark.parametrize( + "missing_hwloc", (None, "all", "include/hwloc.h", "lib/libhwloc.so", "lib/pkgconfig/hwloc.pc") +) +@pytest.mark.parametrize("pkg_config_path", (None, "/custom/lib/pkgconfig")) +def test_build_only_libfabric_and_hwloc( + tmp_path: Path, arch: str, missing_hwloc: str | None, pkg_config_path: str | None +) -> None: + result = _run_installer(tmp_path, missing_hwloc, pkg_config_path, arch) assert result.returncode == 0, result.stderr - args = meson_args.read_text().splitlines() - expected_path = str(efa_path) if with_header and with_library else "" - assert [arg for arg in args if arg.startswith("-Dlibfabric_path=")] == [ - f"-Dlibfabric_path={expected_path}" - ] + configure_args = (tmp_path / "configure_args").read_text().splitlines() + prefix = next( + arg.removeprefix("--prefix=") for arg in configure_args if arg.startswith("--prefix=") + ) + assert Path(prefix).is_relative_to(tmp_path / "scratch") + assert "--disable-rstream" in configure_args + args = (tmp_path / "meson_args").read_text().splitlines() + assert f"-Dlibfabric_path={prefix}" in args expected_pkg_config_path = pkg_config_path or "" - if with_header and with_library and missing_hwloc is None: - expected_pkg_config_path = str(mpi_path / "lib/pkgconfig") + if missing_hwloc is None: + expected_pkg_config_path = str(tmp_path / "mpi/lib/pkgconfig") if pkg_config_path: expected_pkg_config_path += f":{pkg_config_path}" assert (tmp_path / "meson_pkg_config_path").read_text() == expected_pkg_config_path + assert (tmp_path / "restored_ld_path").read_text() == "/existing/lib" + shinit = (tmp_path / "shinit").read_text() + assert f"/lib/{arch}-linux-gnu:" in shinit + assert prefix not in shinit + git_args = (tmp_path / "git_args").read_text().splitlines() + version = git_args[git_args.index("-b") + 1].removeprefix("v") + assert f"nixl=={version}" in (tmp_path / "pip_args").read_text().splitlines() + assert f"nixl-cu13=={version}" in (ROOT / "requirements-dev.txt").read_text().splitlines() + _assert_cleaned(tmp_path) + + +@pytest.mark.parametrize("fail_stage", ("download", "configure", "ninja", "shim")) +def test_failed_install_cleans_build_files(tmp_path: Path, fail_stage: str) -> None: + result = _run_installer(tmp_path, fail_stage=fail_stage) + assert result.returncode != 0 + _assert_cleaned(tmp_path) From 2005b0176e22890c5d0891902fa8fc78c6f61bee Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sun, 1 Nov 2026 00:22:23 +0800 Subject: [PATCH 22/27] Use the configured GitHub mirror for libfabric release downloads Signed-off-by: Yanchao Lu --- docker/common/install_efa_installer.sh | 1 + docker/common/install_nixl.sh | 4 ++-- tests/unittest/scripts/test_install_nixl.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docker/common/install_efa_installer.sh b/docker/common/install_efa_installer.sh index 0fdd241c2c86..f14b8543d15d 100755 --- a/docker/common/install_efa_installer.sh +++ b/docker/common/install_efa_installer.sh @@ -11,6 +11,7 @@ if [ "${ID}" != "ubuntu" ]; then fi EFA_INSTALLER_VERSION="${EFA_INSTALLER_VERSION:-1.51.0}" +# TODO: Use an internal mirror once the pinned EFA installer is available there. EFA_INSTALLER_URL="https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" WORK_DIR="$(mktemp -d)" diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index feb60b0db8a2..9e80f4df7e6f 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -7,7 +7,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Authenticate the github.com clone below; no-op when no token is available. source "${SCRIPT_DIR}/github_auth.sh" -GITHUB_URL="https://github.com" +GITHUB_URL="${GITHUB_MIRROR:-https://github.com}" UCX_INSTALL_PATH="/usr/local/ucx/" # Build against the oldest supported libfabric ABI, independently of the runtime EFA installation. @@ -18,7 +18,7 @@ cd "${WORK_DIR}" LIBFABRIC_VERSION="v1.21.0" LIBFABRIC_INSTALL_PATH="${WORK_DIR}/libfabric" wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ - "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 + "${GITHUB_URL}/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 cd "libfabric-${LIBFABRIC_VERSION#v}" ./autogen.sh diff --git a/tests/unittest/scripts/test_install_nixl.py b/tests/unittest/scripts/test_install_nixl.py index 99cab4b6b5c6..0081b9ce45db 100644 --- a/tests/unittest/scripts/test_install_nixl.py +++ b/tests/unittest/scripts/test_install_nixl.py @@ -15,6 +15,7 @@ # Execute the full installer while replacing network, compilation and package installation. _COMMAND = r""" wget() { + printf '%s\n' "$@" > "$WGET_ARGS" [ "$FAIL_STAGE" != download ] || return 23 : > libfabric.tar.bz2 } @@ -61,6 +62,7 @@ def _run_installer( pkg_config_path: str | None = None, arch: str = "x86_64", fail_stage: str = "", + github_mirror: str | None = None, ) -> subprocess.CompletedProcess[str]: mpi_path = tmp_path / "mpi" for name in ("include/hwloc.h", "lib/libhwloc.so", "lib/pkgconfig/hwloc.pc"): @@ -88,6 +90,7 @@ def _run_installer( "CONFIGURE_ARGS", "PIP_ARGS", "GIT_ARGS", + "WGET_ARGS", "MESON_ARGS", "MESON_PKG_CONFIG_PATH", "RESTORED_LD_PATH", @@ -95,6 +98,8 @@ def _run_installer( env[name] = str(tmp_path / name.lower()) if pkg_config_path is not None: env["PKG_CONFIG_PATH"] = pkg_config_path + if github_mirror is not None: + env["GITHUB_MIRROR"] = github_mirror return subprocess.run( ["bash", "-c", _COMMAND], cwd=tmp_path, @@ -151,3 +156,18 @@ def test_failed_install_cleans_build_files(tmp_path: Path, fail_stage: str) -> N result = _run_installer(tmp_path, fail_stage=fail_stage) assert result.returncode != 0 _assert_cleaned(tmp_path) + + +@pytest.mark.parametrize("github_mirror", (None, "", "https://mirror.example.test/github")) +def test_libfabric_release_mirror(tmp_path: Path, github_mirror: str | None) -> None: + result = _run_installer(tmp_path, github_mirror=github_mirror) + assert result.returncode == 0, result.stderr + base = github_mirror or "https://github.com" + assert ( + f"{base}/ofiwg/libfabric/releases/download/v1.21.0/libfabric-1.21.0.tar.bz2" + in (tmp_path / "wget_args").read_text().splitlines() + ) + # The release-asset mirror does not serve the Git protocol. + assert ( + "https://github.com/ai-dynamo/nixl.git" in (tmp_path / "git_args").read_text().splitlines() + ) From 71803d687f269d7835b85828902f8b208fc22322 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sun, 1 Nov 2026 00:27:03 +0800 Subject: [PATCH 23/27] Bound EFA downloads and verify installer failure stages Signed-off-by: Yanchao Lu --- docker/common/install_efa_installer.sh | 5 +++-- tests/unittest/scripts/test_install_efa_installer.py | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/common/install_efa_installer.sh b/docker/common/install_efa_installer.sh index f14b8543d15d..3e512cd0dbd8 100755 --- a/docker/common/install_efa_installer.sh +++ b/docker/common/install_efa_installer.sh @@ -18,7 +18,8 @@ WORK_DIR="$(mktemp -d)" trap 'rm -rf "${WORK_DIR}"' EXIT cd "${WORK_DIR}" -curl --retry 3 --retry-delay 2 -fsSL -o "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \ +curl --connect-timeout 30 --max-time 1800 --retry 3 --retry-delay 2 \ + -fsSL -o "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \ "${EFA_INSTALLER_URL}" tar -xf "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" cd aws-efa-installer @@ -27,6 +28,6 @@ apt-get update # --skip-mpi: NGC images already ship HPC-X / Open MPI; do not overwrite it. # --skip-rdma-core: use the base image's rdma-core. -# --build-ngc: install the EFA userspace packages without host driver installation, +# --build-ngc: unattended EFA userspace setup without host driver installation, # device verification, limit configuration or dependency replacement. ./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc diff --git a/tests/unittest/scripts/test_install_efa_installer.py b/tests/unittest/scripts/test_install_efa_installer.py index 3b412dedbc23..8d356f2cf7df 100644 --- a/tests/unittest/scripts/test_install_efa_installer.py +++ b/tests/unittest/scripts/test_install_efa_installer.py @@ -64,10 +64,18 @@ def test_efa_container_install( ) assert (result.returncode != 0) == bool(fail_stage), result.stderr assert not list(scratch.iterdir()) + if fail_stage == "download": + assert (tmp_path / "curl_args").exists() + assert not (tmp_path / "efa_args").exists() + elif fail_stage == "install": + assert (tmp_path / "efa_args").exists() if os_id == "rocky": assert not (tmp_path / "curl_args").exists() assert not (tmp_path / "apt_args").exists() elif not fail_stage: + curl_args = (tmp_path / "curl_args").read_text().splitlines() + assert curl_args[curl_args.index("--connect-timeout") + 1] == "30" + assert curl_args[curl_args.index("--max-time") + 1] == "1800" assert (tmp_path / "efa_args").read_text().splitlines() == [ "--skip-mpi", "--skip-rdma-core", From 171b3a5191747bfad4dfaf7cc4ff1bdefaf7e004 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sun, 1 Nov 2026 22:45:52 +0800 Subject: [PATCH 24/27] Complete NIXL LIBFABRIC container dependencies Install GDRCopy 2.5.1 userspace packages when absent and reuse suitable bundled or system hwloc development files. Build and retain hwloc 2.12.2 where needed, with explicit runtime precedence alongside Rocky's system package. Use the official EFA Installer on Rocky8 with existing container options, exclude the temporary DKMS RPM, and select the lib64 runtime directory. Preserve MPI and rdma-core while retaining the Ubuntu installation path. Organize the NIXL installer into build phases, distinguish the temporary libfabric ABI baseline from the runtime, and discover Python bindings via Meson's installation manifest. Document the script-local custom build prefix, runtime library lookup and build dependencies. Remove the installer command-mocking tests. The file tree is identical to the reviewed pre-squash branch. Existing real installer build, import, plugin-loading and MPI initialization checks remain applicable. Full image/L0 and cross-node EFA validation remain pending. Signed-off-by: Yanchao Lu --- docker/Dockerfile.multi | 1 + docker/common/install_efa_installer.sh | 38 ++- docker/common/install_gdrcopy.sh | 66 +++++ docker/common/install_nixl.sh | 244 +++++++++++------- examples/disaggregated/README.md | 55 +++- .../scripts/test_install_efa_installer.py | 88 ------- tests/unittest/scripts/test_install_nixl.py | 173 ------------- 7 files changed, 281 insertions(+), 384 deletions(-) create mode 100644 docker/common/install_gdrcopy.sh delete mode 100644 tests/unittest/scripts/test_install_efa_installer.py delete mode 100644 tests/unittest/scripts/test_install_nixl.py diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index f6747390f76a..2a16bc8e4525 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -114,6 +114,7 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ . /opt/docker/common/github_auth.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_ucx.sh && \ bash /opt/docker/common/install_efa_installer.sh && \ + bash /opt/docker/common/install_gdrcopy.sh && \ GITHUB_MIRROR=${GITHUB_MIRROR} bash /opt/docker/common/install_nixl.sh && \ bash /opt/docker/common/install_etcd.sh && \ rm -rf /root/.cache/uv/archive-v0 && \ diff --git a/docker/common/install_efa_installer.sh b/docker/common/install_efa_installer.sh index 3e512cd0dbd8..52acdb28b674 100755 --- a/docker/common/install_efa_installer.sh +++ b/docker/common/install_efa_installer.sh @@ -3,12 +3,17 @@ # SPDX-License-Identifier: Apache-2.0 set -ex -# Update EFA software stack only on Ubuntu images; skip elsewhere. +# Install the EFA userspace stack on the supported container bases. . /etc/os-release -if [ "${ID}" != "ubuntu" ]; then - echo "Skipping EFA installer: OS '${ID}'." - exit 0 -fi +# NGC PyTorch 26.09 bundles EFA under /opt/amazon/efa/lib; Rocky RPMs use lib64. +case "${ID}:${VERSION_ID}" in + ubuntu:*) EFA_LIB_DIR=/opt/amazon/efa/lib ;; + rocky:8*) EFA_LIB_DIR=/opt/amazon/efa/lib64 ;; + *) + echo "Skipping EFA installer: OS '${ID}' version '${VERSION_ID}'." + exit 0 + ;; +esac EFA_INSTALLER_VERSION="${EFA_INSTALLER_VERSION:-1.51.0}" # TODO: Use an internal mirror once the pinned EFA installer is available there. @@ -24,10 +29,21 @@ curl --connect-timeout 30 --max-time 1800 --retry 3 --retry-delay 2 \ tar -xf "aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" cd aws-efa-installer -apt-get update +# Preserve the base image's MPI and rdma-core. +if [ "${ID}" = ubuntu ]; then + apt-get update + # --build-ngc handles userspace-only installation for Ubuntu containers. + ./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc +else + # Mirror --build-ngc's unattended setup and skip kernel modules, host limits and device checks. + # The standard RPM dependency installation still runs on Rocky. + # Preserve MPI/rdma-core and skip the unrelated NCCL plugin. + # The installer selects top-level DKMS RPMs even with --skip-kmod. + rm -f "RPMS/ROCKYLINUX8/$(uname -m)"/dkms-*.rpm + ./efa_installer.sh --yes --skip-kmod --skip-limit-conf --no-verify \ + --skip-mpi --skip-rdma-core --skip-plugin --disable-ngc +fi -# --skip-mpi: NGC images already ship HPC-X / Open MPI; do not overwrite it. -# --skip-rdma-core: use the base image's rdma-core. -# --build-ngc: unattended EFA userspace setup without host driver installation, -# device verification, limit configuration or dependency replacement. -./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc +# Prefer the EFA runtime over an older distribution libfabric in the loader cache. +printf '%s\n' "${EFA_LIB_DIR}" > /etc/ld.so.conf.d/000_efa.conf +ldconfig diff --git a/docker/common/install_gdrcopy.sh b/docker/common/install_gdrcopy.sh new file mode 100644 index 000000000000..94407fb767c2 --- /dev/null +++ b/docker/common/install_gdrcopy.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +have_headers=false +have_library=false +if printf '#include \n' | cc -E -x c - >/dev/null 2>&1; then + have_headers=true +fi +if ldconfig -p | grep 'libgdrapi\.so\.2 ' >/dev/null; then + have_library=true +fi +if ${have_headers} && ${have_library}; then + echo "GDRCOPY_EXISTING_PRESERVED=1" + exit 0 +fi +if ${have_headers} || ${have_library}; then + echo "Incomplete GDRCopy installation; refusing to overwrite existing files." >&2 + exit 1 +fi + +. /etc/os-release +# Match GDRCopy in the NGC PyTorch 26.09 base images. +GDRCOPY_VERSION=2.5.1 +ARCH=$(uname -m) +# TODO: Add an internal CI mirror after verifying the pinned 2.5.1 DEB/RPM packages. +DOWNLOAD_ROOT="https://developer.download.nvidia.com/compute/redist/gdrcopy/CUDA%2013.0" +WORK_DIR=$(mktemp -d) +trap 'rm -rf "${WORK_DIR}"' EXIT +cd "${WORK_DIR}" + +download() { + curl --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 2 \ + -fsSL "${DOWNLOAD_ROOT}/$1" -o "${1##*/}" + sha256sum "${1##*/}" +} + +case "${ID}:${VERSION_ID}:${ARCH}" in + ubuntu:22.04:x86_64|ubuntu:24.04:x86_64) + package="libgdrapi_${GDRCOPY_VERSION}-1_amd64.Ubuntu${VERSION_ID/./_}.deb" + download "ubuntu${VERSION_ID/./_}/x64/${package}" + dpkg -i "${package}" + ;; + ubuntu:22.04:aarch64|ubuntu:24.04:aarch64) + package="libgdrapi_${GDRCOPY_VERSION}-1_arm64.Ubuntu${VERSION_ID/./_}.deb" + download "ubuntu${VERSION_ID/./_}/aarch64/${package}" + dpkg -i "${package}" + ;; + rocky:8*:x86_64) + runtime="gdrcopy-${GDRCOPY_VERSION}-1.el8.x86_64.rpm" + development="gdrcopy-devel-${GDRCOPY_VERSION}-1.el8.noarch.rpm" + download "rhel8/x64/${runtime}" + download "rhel8/x64/${development}" + rpm -qp --requires "${runtime}" "${development}" + dnf install -y "./${runtime}" "./${development}" + ;; + *) + echo "No GDRCopy ${GDRCOPY_VERSION} package selected for ${ID} ${VERSION_ID} ${ARCH}." >&2 + exit 1 + ;; +esac +ldconfig +printf '#include \n' | cc -E -x c - >/dev/null +ldconfig -p | grep 'libgdrapi\.so\.2 ' +echo "GDRCOPY_USERSPACE_INSTALLED=${GDRCOPY_VERSION}" diff --git a/docker/common/install_nixl.sh b/docker/common/install_nixl.sh index 9e80f4df7e6f..21ab747a3790 100644 --- a/docker/common/install_nixl.sh +++ b/docker/common/install_nixl.sh @@ -7,108 +7,156 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Authenticate the github.com clone below; no-op when no token is available. source "${SCRIPT_DIR}/github_auth.sh" +# Build-only ABI baseline; the EFA installer supplies the runtime libfabric. +LIBFABRIC_BUILD_VERSION="v1.21.0" +# Source-build fallback from NIXL v1.5.2's manylinux dependency set. +HWLOC_VERSION=2.12.2 +NIXL_VERSION="v1.5.2" + GITHUB_URL="${GITHUB_MIRROR:-https://github.com}" +NIXL_REPO="https://github.com/ai-dynamo/nixl.git" +CUDA_PATH="/usr/local/cuda" UCX_INSTALL_PATH="/usr/local/ucx/" +HWLOC_INSTALL_PATH="/usr/local/mpi" # Build against the oldest supported libfabric ABI, independently of the runtime EFA installation. -# Keep the sources and build-only libraries isolated from the caller's working directory. +build_libfabric_for_nixl() { + wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "${GITHUB_URL}/ofiwg/libfabric/releases/download/${LIBFABRIC_BUILD_VERSION}/libfabric-${LIBFABRIC_BUILD_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 + tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 + cd "libfabric-${LIBFABRIC_BUILD_VERSION#v}" + ./autogen.sh + ./configure --prefix="${LIBFABRIC_BUILD_PATH}" \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa \ + --with-cuda="$CUDA_PATH" \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen + make -j"${NPROC:-$(nproc)}" + make install + cd ../ + rm -rf "libfabric-${LIBFABRIC_BUILD_VERSION#v}" +} + +prepare_hwloc() { + # NGC PyTorch's default Open MPI comes from HPC-X and is exposed at /usr/local/mpi. + # Reuse the hwloc bundled with that MPI installation for the EFA plugin. + if [[ -f "${HWLOC_INSTALL_PATH}/include/hwloc.h" && + -f "${HWLOC_INSTALL_PATH}/lib/libhwloc.so" && + -f "${HWLOC_INSTALL_PATH}/lib/pkgconfig/hwloc.pc" ]]; then + export PKG_CONFIG_PATH="${HWLOC_INSTALL_PATH}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + fi + # The LIBFABRIC plugin requires hwloc >= 2.3. For the current base images: + # - NGC PyTorch: reuse bundled HPC-X hwloc 2.7.1 without rebuilding. + # - Non-NGC SBSA Ubuntu 24.04: reuse system hwloc 2.10.0 without rebuilding. + # - Rocky 8: system hwloc 2.2.0 is too old, so build the fallback below. + # Missing hwloc development files also trigger this fallback on any image. + if ! pkg-config --atleast-version=2.3 hwloc; then + curl --connect-timeout 30 --max-time 300 --retry 3 --retry-delay 2 \ + -fsSL "https://download.open-mpi.org/release/hwloc/v${HWLOC_VERSION%.*}/hwloc-${HWLOC_VERSION}.tar.gz" \ + -o hwloc.tar.gz + tar -xzf hwloc.tar.gz + cd "hwloc-${HWLOC_VERSION}" + # Match NIXL's manylinux dependency; retain it for runtime as well as compilation. + ./configure --prefix=/usr/local --libdir=/usr/local/lib --disable-nvml + make -j"${NPROC:-$(nproc)}" + make install + cd .. + rm -rf "hwloc-${HWLOC_VERSION}" hwloc.tar.gz + export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + # On Rocky 8, the system hwloc RPM and its /usr/lib64 files remain installed. + # Prefer the new /usr/local/lib/libhwloc.so.15 through the loader cache. + # This changes default hwloc selection for other consumers, including Open MPI. + printf '%s\n' /usr/local/lib > /etc/ld.so.conf.d/000_nixl_hwloc.conf + ldconfig + fi +} + +build_and_install_nixl() { + OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH + + ARCH_NAME="x86_64-linux-gnu" + GDS_PATH="$CUDA_PATH/targets/x86_64-linux" + if [ "$(uname -m)" != "amd64" ] && [ "$(uname -m)" != "x86_64" ]; then + ARCH_NAME="aarch64-linux-gnu" + GDS_PATH="$CUDA_PATH/targets/sbsa-linux" + fi + + if [ -n "${GITHUB_MIRROR}" ]; then + export PIP_INDEX_URL="https://artifactory.pdx.nvidia.com/artifactory/api/pypi/pypi-remote/simple" + fi + pip3 install meson ninja pybind11 setuptools + + git clone --depth 1 -b ${NIXL_VERSION} ${NIXL_REPO} + cd nixl + + CUDA_SO_PATH=$(find "/usr/local" -name "libcuda.so.1" 2>/dev/null | head -n1) + + if [[ -z "$CUDA_SO_PATH" ]]; then + echo "libcuda.so.1 not found" + exit 1 + fi + + CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) + + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH + # This build-only prefix is removed on exit; runtime libfabric comes from the image or host. + meson setup builddir \ + -Ducx_path=$UCX_INSTALL_PATH \ + -Dlibfabric_path="${LIBFABRIC_BUILD_PATH}" \ + -Dcudapath_lib="$CUDA_PATH/lib64" \ + -Dcudapath_inc="$CUDA_PATH/include" \ + -Dgds_path="$GDS_PATH" \ + -Dinstall_headers=true \ + -Ddisable_plugins=POSIX \ + -Dbuild_tests=false \ + -Dbuild_examples=false \ + --buildtype=release + + cd builddir && ninja install + # Meson uses dist-packages on Ubuntu and versioned site-packages on Rocky. + NIXL_PYTHON_PATH=$(meson introspect --installed | python3 -c ' +import json, pathlib, sys +paths = {str(pathlib.Path(p).parent.parent) for p in json.load(sys.stdin).values() + if pathlib.Path(p).parent.name == "nixl_cu13" and pathlib.Path(p).name.startswith("_bindings.")} +assert len(paths) == 1, paths +print(paths.pop()) + ') + cd ../.. + rm -rf nixl # Remove NIXL source tree to save space + export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH + + # Consumers import `nixl`, but the build above installs the backend as + # `nixl_cu13`. Install the dispatching shim with --no-deps: the backend it would + # otherwise pull from PyPI bundles a second UCX, which segfaults alongside the + # one torch already links. + pip3 install --no-deps "nixl==${NIXL_VERSION#v}" + + echo "export LD_LIBRARY_PATH=/opt/nvidia/nvda_nixl/lib/${ARCH_NAME}:/opt/nvidia/nvda_nixl/lib64:\$LD_LIBRARY_PATH" >> "${ENV}" + # The bindings are outside Python's default search path, so the shim needs PYTHONPATH. This takes + # precedence over the PyPI backend that requirements-dev.txt installs into the test + # environments, so keep that pin in step with NIXL_VERSION above -- otherwise the wheel is + # installed and silently never used. + # Prepend only once: ${ENV} is sourced again by every nested shell. + cat >> "${ENV}" </dev/null | head -n1) - -if [[ -z "$CUDA_SO_PATH" ]]; then - echo "libcuda.so.1 not found" - exit 1 -fi - -CUDA_SO_PATH=$(dirname $CUDA_SO_PATH) - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_SO_PATH -# This build-only prefix is removed on exit; runtime libfabric comes from the image or host. -meson setup builddir \ - -Ducx_path=$UCX_INSTALL_PATH \ - -Dlibfabric_path="${LIBFABRIC_INSTALL_PATH}" \ - -Dcudapath_lib="$CUDA_PATH/lib64" \ - -Dcudapath_inc="$CUDA_PATH/include" \ - -Dgds_path="$GDS_PATH" \ - -Dinstall_headers=true \ - -Ddisable_plugins=POSIX \ - -Dbuild_tests=false \ - -Dbuild_examples=false \ - --buildtype=release - -cd builddir && ninja install -cd ../.. -rm -rf nixl # Remove NIXL source tree to save space -export LD_LIBRARY_PATH=$OLD_LD_LIBRARY_PATH - -# Consumers import `nixl`, but the build above installs the backend as -# `nixl_cu13`. Install the dispatching shim with --no-deps: the backend it would -# otherwise pull from PyPI bundles a second UCX, which segfaults alongside the -# one torch already links. -pip3 install --no-deps "nixl==${NIXL_VERSION#v}" - -echo "export LD_LIBRARY_PATH=/opt/nvidia/nvda_nixl/lib/${ARCH_NAME}:/opt/nvidia/nvda_nixl/lib64:\$LD_LIBRARY_PATH" >> "${ENV}" -# ninja installs the bindings outside site-packages, so the shim needs PYTHONPATH. This takes -# precedence over the PyPI backend that requirements-dev.txt installs into the test -# environments, so keep that pin in step with NIXL_VERSION above -- otherwise the wheel is -# installed and silently never used. -# Prepend only once: ${ENV} is sourced again by every nested shell. -cat >> "${ENV}" <<'EOF' -case ":${PYTHONPATH:-}:" in - *":/opt/nvidia/nvda_nixl/lib/python3/dist-packages:"*) ;; - *) export PYTHONPATH="/opt/nvidia/nvda_nixl/lib/python3/dist-packages${PYTHONPATH:+:${PYTHONPATH}}" ;; -esac -EOF + +build_libfabric_for_nixl +prepare_hwloc +build_and_install_nixl diff --git a/examples/disaggregated/README.md b/examples/disaggregated/README.md index a7f851e59197..6d307c44d664 100644 --- a/examples/disaggregated/README.md +++ b/examples/disaggregated/README.md @@ -52,7 +52,7 @@ If an unsupported NIXL backend is specified, TRT-LLM selects the NIXL UCX backen ### LIBFABRIC Backend Setup -**Container builds:** `docker/common/install_nixl.sh` builds NIXL 1.5.2 and its LIBFABRIC plugin against a temporary libfabric 1.21.0 installation, matching the NIXL upstream wheel's ABI baseline. That installation is removed after the build. Ubuntu builds through `docker/Dockerfile.multi`, including NGC PyTorch images, install the official EFA Installer 1.51.0 userspace libraries at `/opt/amazon/efa` while preserving the existing MPI and rdma-core. Other base images must provide a compatible runtime libfabric and the appropriate provider. Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select the NIXL LIBFABRIC backend; the NIXL UCX backend remains the default. +**Container builds:** `docker/common/install_nixl.sh` builds NIXL 1.5.2 and its LIBFABRIC plugin against a temporary libfabric 1.21.0 installation, matching the NIXL upstream wheel's ABI baseline. That installation is removed after the build. Ubuntu builds, including NGC PyTorch images, and Rocky Linux 8 builds through `docker/Dockerfile.multi` install the official EFA Installer 1.51.0 userspace libraries while preserving the existing MPI and rdma-core. The runtime library directory is `/opt/amazon/efa/lib` on Ubuntu and `/opt/amazon/efa/lib64` on Rocky Linux 8. Rocky builds skip the NCCL plugin installation. Other base images must provide a compatible runtime libfabric and the appropriate provider. Set `TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC` to select the NIXL LIBFABRIC backend; the NIXL UCX backend remains the default. **KV cache V2 over EFA:** Use NIXL 1.5.2 and EFA Installer 1.51.0 or later. These releases include the CUDA VMM context and synchronization fixes, plus EFA completion fixes. The libfabric 1.21.0 build baseline is an ABI compatibility choice, not a guarantee that older runtime libraries support CUDA VMM. `NIXL_DISABLE_CUDA_ADDR_WA=1` is not required for the context-independent allocation fix in NIXL 1.5.2. See the [NIXL release notes](https://github.com/ai-dynamo/nixl/releases/tag/v1.5.2) and [libfabric CUDA VMM fix](https://github.com/ofiwg/libfabric/pull/12918). @@ -69,9 +69,16 @@ If an unsupported NIXL backend is specified, TRT-LLM selects the NIXL UCX backen - For EFA-enabled AWS instances using a non-bundled version, install through the [AWS EFA installer](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html) (recommend using the latest version) **hwloc** -- NGC PyTorch container builds reuse the hwloc development files bundled with HPC-X under `/usr/local/mpi`. When those hwloc development files are present, `docker/common/install_nixl.sh` prepends `/usr/local/mpi/lib/pkgconfig` to `PKG_CONFIG_PATH`; no extra `libhwloc-dev` installation is needed for this configuration. -- hwloc is used to understand the underlying architecture to optimize application performance -- For other base images or custom MPI installations, provide compatible hwloc development files and ensure their `hwloc.pc` is discoverable through `PKG_CONFIG_PATH`. +- NIXL's LIBFABRIC build requires development files for hwloc 2.3 or later. +- NGC PyTorch 26.09 builds reuse HPC-X hwloc 2.7.1 under `/usr/local/mpi`, with `/usr/local/mpi/lib/pkgconfig` prepended to `PKG_CONFIG_PATH`. Non-NGC SBSA Ubuntu 24.04 builds reuse system hwloc 2.10.0. Neither configuration rebuilds hwloc when these development files are present. +- If `pkg-config` cannot find a suitable version, the installer builds and retains hwloc 2.12.2 under `/usr/local`. This applies to Rocky 8's older hwloc 2.2.0. The system RPM and files remain installed, but the loader cache prefers the new library; this also affects other hwloc consumers, including Open MPI. +- Custom installations must provide compatible development and runtime libraries and expose `hwloc.pc` through `PKG_CONFIG_PATH`. + +**GDRCopy (for the source-build installer)** +- The temporary libfabric build enables GDRCopy and requires its userspace library and headers. `docker/Dockerfile.multi` runs `docker/common/install_gdrcopy.sh` to install version 2.5.1 when absent, preserving complete existing installations. Standalone users of `install_nixl.sh` must also provide these dependencies; the GDRCopy helper installs no kernel module. + +**NUMA** +- Provide NUMA development files: `libnuma-dev` on Ubuntu or `numactl-devel` on Rocky Linux. The container base setup installs these packages. **Network Hardware Requirements:** - AWS EFA (Elastic Fabric Adapter), with compatible KV cache settings and dependency versions as described above @@ -89,14 +96,14 @@ Skip these steps if your container already includes the NIXL LIBFABRIC plugin fo 1. **Install libfabric dependencies:** - Follow the installation instructions from the links above based on your system -2. **Provide hwloc development files:** - - Reuse the hwloc bundled with your MPI installation when available, and ensure its `hwloc.pc` is discoverable through `PKG_CONFIG_PATH`. Otherwise, install a compatible hwloc development package before rebuilding NIXL. +2. **Provide source-build dependencies:** + - Provide the hwloc, GDRCopy and NUMA development files described above. Reuse suitable hwloc from your MPI installation when available, and ensure its `hwloc.pc` is discoverable through `PKG_CONFIG_PATH`. 3. **Reinstall NIXL after installing libfabric:** - - After installing libfabric and hwloc, you must rebuild NIXL to generate the LIBFABRIC plugin against your custom path - - You can base your installation on `docker/common/install_nixl.sh`, which always builds a temporary libfabric 1.21.0 installation for the plugin - - For a custom installation, set `LIBFABRIC_INSTALL_PATH=/path/to/libfabric` immediately before the script's `meson setup` command, keeping the other Meson options unchanged - - For more details, see the [NIXL LIBFABRIC Plugin documentation](https://github.com/ai-dynamo/nixl/tree/6ee64753605b3110f8ef96c7cfc2f1315675c9c7/src/plugins/libfabric#nixl-libfabric-plugin) + - To compile the plugin against your custom installation, adapt a local copy of `docker/common/install_nixl.sh`. The unmodified script always builds a temporary libfabric 1.21.0 installation for the plugin. + - Inside `build_and_install_nixl()`, set `LIBFABRIC_BUILD_PATH=/path/to/libfabric` immediately before `meson setup`, which passes `-Dlibfabric_path="${LIBFABRIC_BUILD_PATH}"`. Keep the other Meson options unchanged. Setting the variable only in the environment before launching the script does not override its initialization. + - Keep the custom installation outside `WORK_DIR`, which is deleted on exit. `LIBFABRIC_BUILD_PATH` selects the build dependency only; configure runtime library lookup separately, for example by adding the custom prefix's `lib` or `lib64` directory to `LD_LIBRARY_PATH`, and provide its required fabric provider. + - For more details, see the [NIXL v1.5.2 LIBFABRIC Plugin documentation](https://github.com/ai-dynamo/nixl/blob/v1.5.2/src/plugins/libfabric/README.md#nixl-libfabric-plugin). ##### Option 2: Use Pre-compiled LIBFABRIC Plugin @@ -109,6 +116,7 @@ If you have a pre-compiled `libplugin_LIBFABRIC.so` that matches your NIXL versi export TRTLLM_NIXL_KVCACHE_BACKEND=LIBFABRIC ``` 3. Ensure the plugin was built with the same NIXL version as in your container +4. Ensure the runtime loader can find a compatible libfabric and its provider libraries. `NIXL_PLUGIN_DIR` selects the plugin directory; it does not configure dependency-library lookup. ### NIXL Configuration Examples @@ -349,7 +357,7 @@ Therefore, the command can be written as follows: ```bash # The `container_image` must have the TensorRT-LLM wheel package pre-installed. -# Once the task is successfully launched, an API service will be available externally at http://host_ip:PORT. +# After launch, use the server node's IP address and the configured PORT to reach the API service. # Launch a context with `tp_size=8` using two 4-GPU nodes. srun -A -p -t