Skip to content

Offload cmake cleanups #162196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ZuseZ4 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Offload cmake cleanups #162196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
61bc708
Stop building unused libompdevice code
ZuseZ4 Aug 19, 2026
341d294
Remove LLVM_CONFIG_REAL variables
ZuseZ4 Aug 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/bootstrap/src/core/build_steps/dist.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2898,7 +2898,7 @@ impl CommandLineStep for Offload {
tarball.set_overlay(OverlayKind::Offload);
tarball.is_preview(true);

let omp_offload_libdir = builder.out.join(target).join("offload").join("lib");
let omp_offload_libdir = builder.offload_out(target).join("lib");

@Kobzol Kobzol Sep 2, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be using the path from the omp_offload step output.


for path in omp_offload.artifact_paths_with_symlink_targets() {
let relative = t!(path.strip_prefix(&omp_offload_libdir));
Expand Down
162 changes: 57 additions & 105 deletions src/bootstrap/src/core/build_steps/llvm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1162,10 +1162,7 @@ impl CommandLineStep for RustOffload {

let profile = get_llvm_profile(&builder.config);

cfg.out_dir(&out_dir)
.profile(profile)
.env("LLVM_CONFIG_REAL", llvm_output.llvm_config())
.define("LLVM_DIR", llvm_output.cmake_dir());
cfg.out_dir(&out_dir).profile(profile).define("LLVM_DIR", llvm_output.cmake_dir());

cfg.build();

Expand Down Expand Up @@ -1250,40 +1247,14 @@ impl CommandLineStep for OmpOffload {

let llvm_output = builder.ensure(Llvm { target });

// Running cmake twice in the same folder is known to cause issues, like deleting existing
// binaries. We therefore write our offload artifacts into it's own folder, instead of
// using the llvm build dir.
let out_dir = builder.out.join(self.target.triple).join("offload");
let out_dir = builder.offload_out(self.target);

@Kobzol Kobzol Sep 2, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should hardcode the path here, this is the canonical place where that path is decided.


let mut files = vec![];
let lib_ext = std::env::consts::DLL_EXTENSION;
files.push(out_dir.join("lib").join("libLLVMOffload").with_extension(lib_ext));
files.push(out_dir.join("lib").join("libomp").with_extension(lib_ext));
files.push(out_dir.join("lib").join("libomptarget").with_extension(lib_ext));
files.push(
out_dir.join("lib").join("amdgcn-amd-amdhsa").join("libompdevice").with_extension("a"),
);
files.push(
out_dir
.join("lib")
.join("amdgcn-amd-amdhsa")
.join("libomptarget-amdgpu")
.with_extension("bc"),
);
files.push(
out_dir
.join("lib")
.join("nvptx64-nvidia-cuda")
.join("libompdevice")
.with_extension("a"),
);
files.push(
out_dir
.join("lib")
.join("nvptx64-nvidia-cuda")
.join("libomptarget-nvptx")
.with_extension("bc"),
);
let files = vec![
out_dir.join("lib").join("libLLVMOffload").with_extension(lib_ext),
out_dir.join("lib").join("libomp").with_extension(lib_ext),
out_dir.join("lib").join("libomptarget").with_extension(lib_ext),
];

// Offload/OpenMP are just subfolders of LLVM, so we can use the LLVM sha.
static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
Expand Down Expand Up @@ -1378,79 +1349,61 @@ impl CommandLineStep for OmpOffload {
libstdcxx.parent().map(Path::to_path_buf)
});

// In the context of OpenMP offload, some libraries must be compiled for the gpu target,
// some for the host, and others for both. We do not perform a full cross-compilation, since
// we don't want to run rustc on a GPU.
let omp_targets = vec![target.triple.as_ref(), "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"];
for omp_target in omp_targets {
let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/"));

// If we use an external clang as opposed to building our own llvm_clang, than that clang will
// come with it's own set of default include directories, which are based on a potentially older
// LLVM. This can cause issues, so we overwrite it to include headers based on our
// `src/llvm-project` submodule instead.
let mut cflags = CcFlags::default();
if !builder.config.llvm_clang {
let base = llvm_output.root_dir().join("include");
let inc_dir = base.display();
cflags.push_all(format!(" -I {inc_dir}"));
}
let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/"));

// Logic copied from `configure_llvm`
// ThinLTO is only available when building with LLVM, enabling LLD is required.
// Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin.
let mut ldflags = LdFlags::default();
if builder.config.llvm_thin_lto && !target.contains("apple") {
ldflags.push_all("-fuse-ld=lld");
}
if *omp_target == *target.triple
&& let Some(dir) = &cxx_lib_dir
{
ldflags.push_all(format!("-L{}", dir.display()));
}
// If we use an external clang as opposed to building our own llvm_clang, than that clang will
// come with it's own set of default include directories, which are based on a potentially older
// LLVM. This can cause issues, so we overwrite it to include headers based on our
// `src/llvm-project` submodule instead.
let mut cflags = CcFlags::default();
if !builder.config.llvm_clang {
let base = llvm_output.root_dir().join("include");
let inc_dir = base.display();
cflags.push_all(format!(" -I {inc_dir}"));
}

configure_cmake(builder, target, &mut cfg, true, ldflags, cflags, &[]);

cfg.define("CMAKE_C_COMPILER", &clang)
.define("CMAKE_CXX_COMPILER", &clangxx)
.define("CMAKE_ASM_COMPILER", &clang);

// Re-use the same flags as llvm to control the level of debug information
// generated for offload.
let profile = get_llvm_profile(&builder.config);
trace!(?profile);

// FIXME(offload): Once we move from OMP to Offload (Ol) APIs, we should drop the openmp
// runtime to simplify our build. So far, these are still under development.
cfg.out_dir(&out_dir)
.profile(profile)
.env("LLVM_CONFIG_REAL", llvm_output.llvm_config())
.define("LLVM_ENABLE_ASSERTIONS", "ON")
.define("LLVM_INCLUDE_TESTS", "OFF")
.define("OFFLOAD_INCLUDE_TESTS", "OFF")
.define("LLVM_ROOT", llvm_output.root_dir().join("build"))
.define("LLVM_DIR", llvm_output.cmake_dir())
.define("LLVM_DEFAULT_TARGET_TRIPLE", omp_target);
if let Some(p) = offload_clang_dir.clone() {
cfg.define("Clang_DIR", p);
}
// Logic copied from `configure_llvm`
// ThinLTO is only available when building with LLVM, enabling LLD is required.
// Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin.
let mut ldflags = LdFlags::default();
if builder.config.llvm_thin_lto && !target.contains("apple") {
ldflags.push_all("-fuse-ld=lld");
}

// We don't perform a full cross-compilation of rustc, therefore our target.triple
// will still be a CPU target.
if *omp_target == *target.triple {
// The offload library provides functionality which only makes sense on the host.
cfg.define("LLVM_ENABLE_RUNTIMES", "openmp;offload");
} else {
// OpenMP provides some device libraries, so we also compile it for all gpu targets.
cfg.define("OPENMP_INSTALL_LIBDIR", Path::new("lib").join(omp_target));
cfg.define("LLVM_USE_LINKER", "lld");
cfg.define("LLVM_ENABLE_RUNTIMES", "openmp");
cfg.define("CMAKE_C_COMPILER_TARGET", omp_target);
cfg.define("CMAKE_CXX_COMPILER_TARGET", omp_target);
}
cfg.build();
if let Some(dir) = &cxx_lib_dir {
ldflags.push_all(format!("-L{}", dir.display()));
}

configure_cmake(builder, target, &mut cfg, true, ldflags, cflags, &[]);

cfg.define("CMAKE_C_COMPILER", &clang)
.define("CMAKE_CXX_COMPILER", &clangxx)
.define("CMAKE_ASM_COMPILER", &clang);

// Re-use the same flags as llvm to control the level of debug information
// generated for offload.
let profile = get_llvm_profile(&builder.config);
trace!(?profile);

// FIXME(offload): Once we move from OMP to Offload (Ol) APIs, we should drop the openmp
// runtime to simplify our build. So far, these are still under development.
cfg.out_dir(&out_dir)
.profile(profile)
.define("LLVM_ENABLE_ASSERTIONS", "ON")
.define("LLVM_INCLUDE_TESTS", "OFF")
.define("OFFLOAD_INCLUDE_TESTS", "OFF")
.define("LLVM_ROOT", llvm_output.root_dir().join("build"))
.define("LLVM_DIR", llvm_output.cmake_dir())
.define("LLVM_DEFAULT_TARGET_TRIPLE", &*target.triple);
if let Some(p) = offload_clang_dir {
cfg.define("Clang_DIR", p);
}

// The offload library provides functionality which only makes sense on the host.
cfg.define("LLVM_ENABLE_RUNTIMES", "openmp;offload");

cfg.build();

t!(stamp.write());

for p in &files {
Expand Down Expand Up @@ -1593,7 +1546,6 @@ impl CommandLineStep for Enzyme {

cfg.out_dir(&out_dir)
.profile(profile)
.env("LLVM_CONFIG_REAL", llvm_output.llvm_config())
.define("LLVM_ENABLE_ASSERTIONS", "ON")
.define("ENZYME_EXTERNAL_SHARED_LIB", "ON")
.define("ENZYME_BC_LOADER", "OFF")
Expand Down
8 changes: 8 additions & 0 deletions src/bootstrap/src/core/builder/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1524,6 +1524,14 @@ Alternatively, you can set `build.local-rebuild=true` and use a stage0 compiler
self.ensure(llvm::Llvm { target: self.host_target }).llvm_config().to_owned()
}

/// Root output directory of the OpenMP/Offload runtimes for `target`
///
/// Deliberately not under `llvm_output_dir`, since running cmake twice in the same folder is
/// known to cause issues, like deleting existing binaries.
pub fn offload_out(&self, target: TargetSelection) -> PathBuf {

@Kobzol Kobzol Sep 2, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I deliberately removed these functions recently, because they are an antipattern 😅 Steps should be ensuring other steps to get the build directory, rather than implicitly depending on paths from a shared function.

self.out.join(target).join("offload")
}

/// Updates all submodules, and exits with an error if submodule
/// management is disabled and the submodule does not exist.
pub fn require_and_update_all_submodules(&self) {
Expand Down
4 changes: 4 additions & 0 deletions src/doc/rustc-dev-guide/src/offload/installation.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ cd rust
./configure --enable-llvm-link-shared --release-channel=nightly --enable-llvm-assertions --enable-llvm-offload --enable-llvm-enzyme --enable-clang --enable-lld --enable-option-checking --enable-ninja --disable-docs
```

If you would rather reuse an existing clang than build one, drop `--enable-clang` and pass
`--enable-llvm-offload-clang-dir=<absolute path to the directory holding ClangConfig.cmake>`
instead. It should match the (major version of the) LLVM in `src/llvm-project`.

Afterwards you can build rustc using:
```console
./x build --stage 1 library
Expand Down
Loading