From cc3ec7072681349fc82c97c48272ef47732c4621 Mon Sep 17 00:00:00 2001 From: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:42:09 -0700 Subject: [PATCH 1/4] fix(nvjitlink-sys): find libnvJitLink under lib/ as well as lib64/ nvjitlink-sys probed only `/lib64/libnvJitLink.so` under each CUDA root. conda environments ship the library under `/lib/` and have no `lib64/`, so the probe fell through to loading by SONAME. A library found that way cannot be tied to an exact file, and `--materialize-cubin` then refused to run with CUDA_HOME set to the environment. libNVVM, under `/nvvm/lib64/`, was found in the same layout. Probe `lib64/` then `lib/` under each root, and say in the materialization error that a library found only by SONAME has no exact file, and how to point at one. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> --- .../cargo-oxide/src/commands/materialize.rs | 2 +- crates/nvjitlink-sys/src/lib.rs | 31 +++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/cargo-oxide/src/commands/materialize.rs b/crates/cargo-oxide/src/commands/materialize.rs index fb49d68aa6..d3b8372eac 100644 --- a/crates/cargo-oxide/src/commands/materialize.rs +++ b/crates/cargo-oxide/src/commands/materialize.rs @@ -310,7 +310,7 @@ pub fn print_materializer_handshake() { }); let handshake = finalizer.materializer_handshake().unwrap_or_else(|| { eprintln!( - "the loaded libNVVM or nvJitLink library cannot be tied to an exact file; refusing materialization because Cargo could not fingerprint the compiler inputs" + "the loaded libNVVM or nvJitLink library cannot be tied to an exact file; refusing materialization because Cargo could not fingerprint the compiler inputs. A library found only by SONAME has no exact file: set LIBNVVM_PATH and LIBNVJITLINK_PATH to the library files, or CUDA_HOME to a toolkit root that contains them" ); std::process::exit(1); }); diff --git a/crates/nvjitlink-sys/src/lib.rs b/crates/nvjitlink-sys/src/lib.rs index 4ff5f279fc..4c1449bbd9 100644 --- a/crates/nvjitlink-sys/src/lib.rs +++ b/crates/nvjitlink-sys/src/lib.rs @@ -7,7 +7,8 @@ //! //! nvJitLink links one or more LTOIR modules (and other input forms) into //! a final cubin or PTX. It is part of the CUDA Toolkit and ships at -//! `/lib64/libnvJitLink.so`. +//! `/lib64/libnvJitLink.so`, or at `/lib/libnvJitLink.so` in +//! layouts without `lib64`, such as conda environments. //! //! # Symbol naming //! @@ -729,8 +730,7 @@ fn open_library(tried: &mut Vec, retain_exact_file: bool) -> Option Option Vec { + roots + .iter() + .flat_map(|root| ["lib64", "lib"].map(|dir| root.join(dir).join("libnvJitLink.so"))) + .collect() +} + fn cuda_roots() -> Vec { cuda_roots_from_env(|var| std::env::var(var).ok()) } @@ -1082,6 +1093,20 @@ mod tests { assert_eq!(prepared.as_ref(), auto_detected); } + #[test] + fn library_candidates_try_lib64_then_lib_under_each_root() { + let roots = [PathBuf::from("/cuda"), PathBuf::from("/conda")]; + assert_eq!( + library_candidates(&roots), + vec![ + PathBuf::from("/cuda/lib64/libnvJitLink.so"), + PathBuf::from("/cuda/lib/libnvJitLink.so"), + PathBuf::from("/conda/lib64/libnvJitLink.so"), + PathBuf::from("/conda/lib/libnvJitLink.so"), + ] + ); + } + #[test] fn cuda_roots_prefers_project_toolkit_env_var() { let roots = cuda_roots_from_env(|var| match var { From 92c0bf800b4dd4de1dd40b528c3530116e3cb059 Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 16:27:37 +0530 Subject: [PATCH 2/4] test(nvjitlink-sys): cover lib/ discovery under a CUDA root Hoist the per-root probe loop out of open_library into open_library_under_roots, which takes the roots as a slice. The LIBNVJITLINK_PATH override and the SONAME fallback stay in open_library, so a developer's environment cannot short-circuit a test of the loop. Add three Linux tests that build the probe DSO under a temp root: a lib/-only root is found and its retained identity is the lib/ file, lib64/ wins when both directories exist, and an empty root reports both candidates in the tried list. The first and the third fail against the lib64-only probe that main had. Name conda-forge and nixpkgs as the lib/ layouts in the crate docs, list the discovery order there, and record that only the unversioned libnvJitLink.so is probed under a root. Signed-off-by: nihalpasham --- cuda-oxide/crates/nvjitlink-sys/src/lib.rs | 152 +++++++++++++++++++-- 1 file changed, 142 insertions(+), 10 deletions(-) diff --git a/cuda-oxide/crates/nvjitlink-sys/src/lib.rs b/cuda-oxide/crates/nvjitlink-sys/src/lib.rs index 4c1449bbd9..c70606565d 100644 --- a/cuda-oxide/crates/nvjitlink-sys/src/lib.rs +++ b/cuda-oxide/crates/nvjitlink-sys/src/lib.rs @@ -7,8 +7,24 @@ //! //! nvJitLink links one or more LTOIR modules (and other input forms) into //! a final cubin or PTX. It is part of the CUDA Toolkit and ships at -//! `/lib64/libnvJitLink.so`, or at `/lib/libnvJitLink.so` in -//! layouts without `lib64`, such as conda environments. +//! `/lib64/libnvJitLink.so`. conda-forge and nixpkgs lay the toolkit +//! out with `lib/` instead of `lib64/`, so both directories are probed. +//! +//! # Library discovery +//! +//! [`LibNvJitLink::load`] tries (in order): +//! 1. `LIBNVJITLINK_PATH` env var, if set. +//! 2. `/lib64/libnvJitLink.so`, then `/lib/libnvJitLink.so`, for +//! `` in `CUDA_TOOLKIT_PATH`, `CUDA_HOME`, `CUDA_PATH`, +//! `/usr/local/cuda`, `/opt/cuda`. +//! 3. The system loader (`libnvJitLink.so.13`, `libnvJitLink.so.12`, +//! `libnvJitLink.so`). +//! +//! Only the unversioned `libnvJitLink.so` name is probed under a root. A +//! layout that ships only `libnvJitLink.so.13` is reached through the system +//! loader, which does not say which file it mapped, so that load cannot be +//! tied to one file on disk. Set `LIBNVJITLINK_PATH` to the file when the +//! exact file matters, as it does for cubin materialization. //! //! # Symbol naming //! @@ -730,11 +746,8 @@ fn open_library(tried: &mut Vec, retain_exact_file: bool) -> Option, retain_exact_file: bool) -> Option, + retain_exact_file: bool, +) -> Option { + for path in library_candidates(roots) { + tried.push(path.display().to_string()); + if let Some(opened) = open_library_path(&path, retain_exact_file) { + return Some(opened); + } + } + None +} + fn open_library_path(path: &Path, retain_exact_file: bool) -> Option { #[cfg(not(target_os = "linux"))] let _ = retain_exact_file; @@ -793,9 +825,13 @@ fn open_library_path(path: &Path, retain_exact_file: bool) -> Option Vec { roots .iter() @@ -933,6 +969,102 @@ mod tests { std::fs::remove_dir_all(directory).unwrap(); } + /// A fresh directory under the system temp dir, used as a CUDA root. + #[cfg(target_os = "linux")] + fn scratch_root(tag: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "nvjitlink-sys-{tag}-{}-{nonce}", + std::process::id() + )); + std::fs::create_dir(&root).unwrap(); + root + } + + /// Build the probe DSO as `//libnvJitLink.so`. + #[cfg(target_os = "linux")] + fn place_probe_library(root: &Path, dir: &str, value: i32) -> PathBuf { + let directory = root.join(dir); + std::fs::create_dir(&directory).unwrap(); + let library_path = directory.join("libnvJitLink.so"); + compile_probe_library(&directory.join("probe.c"), &library_path, value); + library_path + } + + #[cfg(target_os = "linux")] + fn probe_value(opened: &OpenedLibrary) -> c_int { + let probe: Symbol c_int> = + unsafe { opened.library.get(b"cuda_oxide_probe") }.unwrap(); + unsafe { probe() } + } + + #[cfg(target_os = "linux")] + fn file_identity(path: &Path) -> Option { + LibraryFileIdentity::capture_file(&File::open(path).unwrap()) + } + + #[cfg(target_os = "linux")] + fn candidate_strings(root: &Path, dirs: &[&str]) -> Vec { + dirs.iter() + .map(|dir| root.join(dir).join("libnvJitLink.so").display().to_string()) + .collect() + } + + #[cfg(target_os = "linux")] + #[test] + fn root_probe_loads_lib_when_lib64_is_absent() { + let root = scratch_root("root-lib"); + let library_path = place_probe_library(&root, "lib", 7); + let expected_identity = file_identity(&library_path); + assert!(expected_identity.is_some()); + + let mut tried = Vec::new(); + let opened = open_library_under_roots(std::slice::from_ref(&root), &mut tried, true) + .expect("lib/libnvJitLink.so is found when lib64/ is absent"); + + assert_eq!(probe_value(&opened), 7); + assert_eq!(opened.loaded_identity, expected_identity); + assert_eq!(tried, candidate_strings(&root, &["lib64", "lib"])); + + drop(opened); + std::fs::remove_dir_all(root).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn root_probe_prefers_lib64_over_lib() { + let root = scratch_root("root-both"); + let lib64_path = place_probe_library(&root, "lib64", 1); + place_probe_library(&root, "lib", 2); + let lib64_identity = file_identity(&lib64_path); + + let mut tried = Vec::new(); + let opened = open_library_under_roots(std::slice::from_ref(&root), &mut tried, true) + .expect("lib64/libnvJitLink.so is found"); + + assert_eq!(probe_value(&opened), 1); + assert_eq!(opened.loaded_identity, lib64_identity); + assert_eq!(tried, candidate_strings(&root, &["lib64"])); + + drop(opened); + std::fs::remove_dir_all(root).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn root_probe_reports_both_candidates_for_an_empty_root() { + let root = scratch_root("root-empty"); + + let mut tried = Vec::new(); + assert!(open_library_under_roots(std::slice::from_ref(&root), &mut tried, true).is_none()); + assert_eq!(tried, candidate_strings(&root, &["lib64", "lib"])); + + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn open_descriptor_remains_bound_to_replaced_inode() { let nonce = SystemTime::now() From d4212f145d9a090a64790c49bd09251b43c9b39b Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 16:27:37 +0530 Subject: [PATCH 3/4] fix(cargo-oxide): name every discovery variable in the materialization refusal The refusal fires whenever either tool has no exact file: after a SONAME load, but also when the /proc/self/fd load falls back to a pathname load, when the digest is missing, or when the file changed. Call the SONAME load the usual cause rather than the only one, and list CUDA_TOOLKIT_PATH before CUDA_HOME, as the sys crates' own errors do. Signed-off-by: nihalpasham --- cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs b/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs index d3b8372eac..80dee3ee40 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs @@ -310,7 +310,7 @@ pub fn print_materializer_handshake() { }); let handshake = finalizer.materializer_handshake().unwrap_or_else(|| { eprintln!( - "the loaded libNVVM or nvJitLink library cannot be tied to an exact file; refusing materialization because Cargo could not fingerprint the compiler inputs. A library found only by SONAME has no exact file: set LIBNVVM_PATH and LIBNVJITLINK_PATH to the library files, or CUDA_HOME to a toolkit root that contains them" + "the loaded libNVVM or nvJitLink library cannot be tied to an exact file; refusing materialization because Cargo could not fingerprint the compiler inputs. The usual cause is a library found only by SONAME: set LIBNVVM_PATH or LIBNVJITLINK_PATH to the library file, or CUDA_TOOLKIT_PATH or CUDA_HOME to a toolkit root that contains both" ); std::process::exit(1); }); From a00a3da132dd93b90517c9f675a522a3999c5636 Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 16:27:37 +0530 Subject: [PATCH 4/4] docs(cuda-oxide): describe the lib64 then lib nvJitLink probe order The nvjitlink-sys README, the cuda-host ltoir module docs and the cargo oxide doctor hint still said lib64 only. State the lib64-then-lib order, name conda-forge and nixpkgs as the lib/ layouts, and record that only the unversioned libnvJitLink.so is probed under a root, with LIBNVJITLINK_PATH as the way to name a versioned file. Signed-off-by: nihalpasham --- cuda-oxide/crates/cargo-oxide/src/commands/doctor.rs | 3 ++- cuda-oxide/crates/cuda-host/src/ltoir.rs | 4 ++-- cuda-oxide/crates/nvjitlink-sys/README.md | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/doctor.rs b/cuda-oxide/crates/cargo-oxide/src/commands/doctor.rs index 4af245bcb6..143d7e979b 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/doctor.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/doctor.rs @@ -568,7 +568,8 @@ pub fn doctor(ctx: &Context) { Err(e) => { println!("✗ {}", e); eprintln!(" Required only when kernels call CUDA libdevice math."); - eprintln!(" Ships with the CUDA Toolkit at /lib64/libnvJitLink.so."); + eprintln!(" Ships with the CUDA Toolkit at /lib64/libnvJitLink.so,"); + eprintln!(" or /lib/ in conda-forge and nixpkgs layouts."); ok = false; } } diff --git a/cuda-oxide/crates/cuda-host/src/ltoir.rs b/cuda-oxide/crates/cuda-host/src/ltoir.rs index 810c56a396..5eaad7ef43 100644 --- a/cuda-oxide/crates/cuda-host/src/ltoir.rs +++ b/cuda-oxide/crates/cuda-host/src/ltoir.rs @@ -33,8 +33,8 @@ //! - **libNVVM**: `LIBNVVM_PATH`, then `/nvvm/lib64/libnvvm.so` for //! `` in `CUDA_TOOLKIT_PATH`, `CUDA_HOME`, `CUDA_PATH`, //! `/usr/local/cuda`, `/opt/cuda`, then the system loader. -//! - **nvJitLink**: the same order, using `LIBNVJITLINK_PATH` and -//! `/lib64/libnvJitLink.so`. +//! - **nvJitLink**: the same order, using `LIBNVJITLINK_PATH`, then +//! `/lib64/libnvJitLink.so` and `/lib/libnvJitLink.so`. //! - **libdevice**: `CUDA_OXIDE_LIBDEVICE` env var, then //! `/nvvm/libdevice/libdevice.10.bc` for the same roots. //! - **Artifact target and options**: the emitted `.target` records the diff --git a/cuda-oxide/crates/nvjitlink-sys/README.md b/cuda-oxide/crates/nvjitlink-sys/README.md index 8368fd4e13..446b16a1a2 100644 --- a/cuda-oxide/crates/nvjitlink-sys/README.md +++ b/cuda-oxide/crates/nvjitlink-sys/README.md @@ -18,10 +18,10 @@ None. The library is loaded at runtime, so the CUDA Toolkit only needs to be pre `LibNvJitLink::load()` tries (in order): 1. `LIBNVJITLINK_PATH` env var, if set. -2. `/lib64/libnvJitLink.so` for `` in `CUDA_TOOLKIT_PATH`, `CUDA_HOME`, `CUDA_PATH`, `/usr/local/cuda`, `/opt/cuda`. +2. `/lib64/libnvJitLink.so`, then `/lib/libnvJitLink.so`, for `` in `CUDA_TOOLKIT_PATH`, `CUDA_HOME`, `CUDA_PATH`, `/usr/local/cuda`, `/opt/cuda`. 3. The system loader (`libnvJitLink.so.13`, `libnvJitLink.so.12`, `libnvJitLink.so`). -nvJitLink ships with the standard CUDA Toolkit at `/lib64/`. No separate download. +nvJitLink ships with the standard CUDA Toolkit at `/lib64/`; conda-forge and nixpkgs layouts place it at `/lib/`. No separate download. Only the unversioned `libnvJitLink.so` name is probed under a root. A layout that ships only `libnvJitLink.so.13` is found through the system loader instead, and such a load cannot be tied to one file on disk, which cubin materialization (`cargo oxide build --materialize-cubin`) requires. Set `LIBNVJITLINK_PATH` to the versioned file in that case. ## Symbol naming