From 8c4a42e9c95b2a6bdc14a7531904d8469f0e7ca6 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Mon, 19 Oct 2026 22:33:32 -0700 Subject: [PATCH] Preserve convergence on native device extern declarations Carry the convergence marker through private dependency metadata and emit it on each marked LLVM declaration before native optimization. Deduplicate repeated extern symbols while rejecting incompatible signatures and convergence requirements. Signed-off-by: Trent Nelson --- cuda-oxide/crates/cuda-macros/src/lib.rs | 79 ++++--------- .../crates/cuda-oxide-codegen/src/export.rs | 16 +-- .../crates/cuda-oxide-codegen/src/lower.rs | 62 +++++++++- .../crates/llvm-export/src/export/externs.rs | 16 +-- .../crates/llvm-export/src/export/module.rs | 28 +++-- .../tests/export_test/device_externs.rs | 107 ++++++++++++++++++ .../crates/reserved-oxide-symbols/src/lib.rs | 5 + .../examples/device_ffi_test/Cargo.lock | 8 ++ .../examples/device_ffi_test/Cargo.toml | 1 + .../examples/device_ffi_test/README.md | 60 ++++------ .../convergence-helper/Cargo.toml | 12 ++ .../convergence-helper/src/lib.rs | 22 ++++ .../examples/device_ffi_test/src/main.rs | 18 +-- .../device_ffi_test/verify-code-shape.sh | 23 +++- .../rustc-codegen-cuda/src/collector.rs | 26 ++--- 15 files changed, 327 insertions(+), 156 deletions(-) create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs diff --git a/cuda-oxide/crates/cuda-macros/src/lib.rs b/cuda-oxide/crates/cuda-macros/src/lib.rs index 0a3d738b55..75afea241f 100644 --- a/cuda-oxide/crates/cuda-macros/src/lib.rs +++ b/cuda-oxide/crates/cuda-macros/src/lib.rs @@ -1015,10 +1015,14 @@ pub fn tile_export(attr: TokenStream, item: TokenStream) -> TokenStream { // NVVM Attributes for Device FFI // ============================================================================ -/// Marks a device function as convergent. +/// Marks an external device function declaration as convergent. /// -/// Convergent functions must be called by all threads in a warp/block together. -/// This prevents the optimizer from moving calls across control flow boundaries. +/// LLVM forbids transformations that add control dependencies to the call, +/// including before external device definitions are linked. The annotation +/// does not make threads reach the call or guarantee reconvergence. Callers +/// must satisfy the function's warp or block participation requirements. +/// Every declaration of the same symbol reachable in a device module must +/// agree on this annotation; conflicting declarations are rejected. /// /// # When to Use /// @@ -1039,77 +1043,32 @@ pub fn tile_export(attr: TokenStream, item: TokenStream) -> TokenStream { /// # Generated LLVM IR /// /// ```llvm -/// declare float @cub_block_reduce_sum(float, ptr) #0 -/// attributes #0 = { convergent nounwind } +/// declare float @cub_block_reduce_sum(float, ptr) convergent /// ``` #[proc_macro_attribute] pub fn convergent(_attr: TokenStream, item: TokenStream) -> TokenStream { - // This is a marker attribute - just pass through the item unchanged. - // The collector will read this attribute and apply the LLVM convergent attribute. - item + // The procedural attribute disappears during expansion. A doc alias is + // retained in metadata even for private foreign items, unlike doc comments. + let marker = reserved_oxide_symbols::DEVICE_EXTERN_CONVERGENT_MARKER; + let item = proc_macro2::TokenStream::from(item); + quote::quote!(#[doc(alias = #marker)] #item).into() } -/// Marks a device function as pure (no side effects). -/// -/// Pure functions only depend on their inputs and have no side effects. -/// This enables aggressive optimizations like CSE and dead code elimination. -/// -/// # When to Use -/// -/// - Math functions that don't access memory -/// - Functions that compute results purely from input arguments +/// Compatibility annotation for a pure device function. /// -/// # Example -/// -/// ```ignore -/// #[device] -/// extern "C" { -/// #[pure] -/// fn fast_rsqrt(x: f32) -> f32; -/// } -/// ``` -/// -/// # Generated LLVM IR -/// -/// ```llvm -/// declare float @fast_rsqrt(float) #0 -/// attributes #0 = { nounwind readnone } -/// ``` +/// This annotation is currently inactive: it records no compiler effect and +/// emits no LLVM memory-effect attributes. #[proc_macro_attribute] pub fn pure(_attr: TokenStream, item: TokenStream) -> TokenStream { - // Marker attribute - collector will read and apply LLVM readnone attribute item } -/// Marks a device function as read-only. +/// Compatibility annotation for a read-only device function. /// -/// Read-only functions may read memory but never write to it. -/// This enables optimizations like load hoisting and caching. -/// -/// # When to Use -/// -/// - Lookup table functions -/// - Functions that only read from input arrays -/// -/// # Example -/// -/// ```ignore -/// #[device] -/// extern "C" { -/// #[readonly] -/// fn lookup_table(table: *const f32, idx: i32) -> f32; -/// } -/// ``` -/// -/// # Generated LLVM IR -/// -/// ```llvm -/// declare float @lookup_table(ptr, i32) #0 -/// attributes #0 = { nounwind readonly } -/// ``` +/// This annotation is currently inactive: it records no compiler effect and +/// emits no LLVM memory-effect attributes. #[proc_macro_attribute] pub fn readonly(_attr: TokenStream, item: TokenStream) -> TokenStream { - // Marker attribute - collector will read and apply LLVM readonly attribute item } diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs index 12fe5ecda2..fc1b5521db 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs @@ -44,25 +44,19 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on our -/// declarations redundant. -/// -/// This struct is retained for the pipeline API but values are not used in code generation. +/// Convergence is emitted on each marked declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). - /// NOTE: Not currently emitted to LLVM IR. + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, /// Function is pure (no side effects, result depends only on inputs). - /// NOTE: Not currently emitted to LLVM IR. + /// Inactive; not emitted to LLVM IR. pub is_pure: bool, /// Function is read-only (only reads memory, doesn't write). - /// NOTE: Not currently emitted to LLVM IR. + /// Inactive; not emitted to LLVM IR. pub is_readonly: bool, } diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs index 05011e5e58..d08f68f8d5 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs @@ -84,7 +84,7 @@ pub fn add_device_extern_declarations( region.iter(ctx).next().expect("Module should have a block") }; - let declared_symbols: HashMap<_, _> = block + let mut declared_symbols: HashMap<_, _> = block .deref(ctx) .iter(ctx) .filter_map(|op| { @@ -140,6 +140,7 @@ pub fn add_device_extern_declarations( // Insert at the front of the module (declarations come before definitions) func_op.get_operation().insert_at_front(block, ctx); + declared_symbols.insert(decl.export_name.clone(), func_type); } Ok(()) @@ -196,3 +197,62 @@ fn device_extern_type_to_pliron( } }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::export::DeviceExternAttrs; + use llvm_export::export::{NvvmExportConfig, export_module_with_externs}; + use pliron::builtin::ops::ModuleOp; + use pliron::common_traits::Verify; + + #[test] + fn repeated_externs_keep_one_operation_and_preserve_conflict_checks() { + let mut ctx = Context::new(); + let module = ModuleOp::new(&mut ctx, "externs".try_into().unwrap()); + let declaration = DeviceExternDecl { + export_name: "external".to_string(), + param_types: vec![DeviceExternType::Integer(32)], + return_type: DeviceExternType::Void, + attrs: DeviceExternAttrs::default(), + }; + let mut externs = [declaration.clone(), declaration]; + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap(); + let declarations = { + let region = module.get_operation().deref(&ctx).get_region(0).deref(&ctx); + let block = region.iter(&ctx).next().unwrap(); + block + .deref(&ctx) + .iter(&ctx) + .filter(|&op| { + Operation::get_op::(op, &ctx).is_some_and( + |function| function.get_symbol_name(&ctx).to_string() == "external", + ) + }) + .count() + }; + assert_eq!(declarations, 1); + module.get_operation().deref(&ctx).verify(&ctx).unwrap(); + let ir = export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .unwrap(); + assert_eq!(ir.matches("declare void @external(i32)").count(), 1); + + // Lowering shares only the FuncOp. The complete declarations still + // reach export, where differing convergence requirements are rejected. + externs[1].attrs.is_convergent = true; + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap(); + let error = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .unwrap_err(); + assert!(error.contains("convergent is false versus true"), "{error}"); + + externs[1].param_types[0] = DeviceExternType::Float32; + let error = + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap_err(); + assert!( + error + .to_string() + .contains("conflicts with the call-site declaration") + ); + } +} diff --git a/cuda-oxide/crates/llvm-export/src/export/externs.rs b/cuda-oxide/crates/llvm-export/src/export/externs.rs index ceeab494f0..2cea5a5fc0 100644 --- a/cuda-oxide/crates/llvm-export/src/export/externs.rs +++ b/cuda-oxide/crates/llvm-export/src/export/externs.rs @@ -225,22 +225,18 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on -/// declarations redundant. -/// -/// This struct is retained for potential future use or for debugging/inspection. +/// Convergence is emitted on the declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive; they are not inferred from external +/// definitions that become available only during LTO. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, - /// Function is pure (no side effects). Maps to LLVM `readnone`. + /// Function is pure (no side effects). Inactive for Rust source annotations. pub is_pure: bool, - /// Function is read-only (only reads memory). Maps to LLVM `readonly`. + /// Function is read-only (only reads memory). Inactive for Rust source annotations. pub is_readonly: bool, } diff --git a/cuda-oxide/crates/llvm-export/src/export/module.rs b/cuda-oxide/crates/llvm-export/src/export/module.rs index 1a9e934851..9d425d54c4 100644 --- a/cuda-oxide/crates/llvm-export/src/export/module.rs +++ b/cuda-oxide/crates/llvm-export/src/export/module.rs @@ -487,10 +487,17 @@ fn index_device_externs( validate_device_extern_function_shape(state, decl)?; if let Some(previous) = state.device_externs.get(&decl.export_name) { if previous != decl { - return Err(format!( + let conflict = format!( "conflicting device-extern declarations for `@{}`", decl.export_name - )); + ); + if previous.attrs.is_convergent != decl.attrs.is_convergent { + return Err(format!( + "{conflict}: convergent is {} versus {}", + previous.attrs.is_convergent, decl.attrs.is_convergent + )); + } + return Err(conflict); } continue; } @@ -610,12 +617,9 @@ pub(super) fn export_module_with_externs_impl( // 2. Device extern declarations (before function definitions) // - // NOTE: We intentionally do NOT emit LLVM attributes on these declarations. - // The external LTOIR (from nvcc -dc -dlto) already contains proper attributes - // (convergent, nounwind, memory, etc.) on the function DEFINITIONS. - // When nvJitLink performs LTO linking, it uses the definition's attributes. - // Attributes on declarations are redundant and were causing issues where - // all externs incorrectly got the same attribute group. + // Convergence must be visible before libNVVM optimizes this module, not + // only when external definitions arrive at LTO. Emit it per declaration + // so unrelated externs do not inherit a shared attribute group. if !device_externs.is_empty() { writeln!( &mut output, @@ -643,7 +647,11 @@ pub(super) fn export_module_with_externs_impl( } param.write_llvm_with_attr(&mut output, state.legacy_typed_pointers())?; } - writeln!(&mut output, ")").unwrap(); + write!(&mut output, ")").unwrap(); + if decl.attrs.is_convergent { + write!(&mut output, " convergent").unwrap(); + } + writeln!(&mut output).unwrap(); } writeln!(&mut output).unwrap(); } @@ -711,7 +719,7 @@ pub(super) fn export_module_with_externs_impl( } // 6. Emit attribute groups for convergent intrinsics used by module functions - // Note: Device extern declarations no longer get attribute groups - see section 2 comment. + // Device extern convergence is emitted inline, independently of this group. if state.convergent_used { writeln!(&mut output).unwrap(); writeln!(&mut output, "attributes #0 = {{ convergent }}").unwrap(); diff --git a/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs b/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs index ee928c3819..f68a227611 100644 --- a/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs +++ b/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs @@ -23,6 +23,113 @@ use pliron::{ use crate::common::module_top_block; +fn mixed_convergence_module(ctx: &mut Context) -> (ModuleOp, Vec) { + let module = ModuleOp::new(ctx, "mixed_convergence".try_into().unwrap()); + let module_block = module_top_block(ctx, &module); + let ptr_ty = PointerType::get(ctx, 0); + let void_ty = VoidType::get(ctx); + let external_ty = FuncType::get(ctx, void_ty.into(), vec![ptr_ty.into()], false); + let caller = FuncOp::new(ctx, "caller".try_into().unwrap(), external_ty); + let entry = caller.get_or_create_entry_block(ctx); + let pointer = entry.deref(ctx).get_argument(0); + let mut externs = Vec::new(); + for (name, is_convergent) in [("collective", true), ("ordinary", false)] { + FuncOp::new(ctx, name.try_into().unwrap(), external_ty) + .get_operation() + .insert_at_back(module_block, ctx); + CallOp::new( + ctx, + CallOpCallable::Direct(name.try_into().unwrap()), + external_ty, + vec![pointer], + ) + .get_operation() + .insert_at_back(entry, ctx); + externs.push(DeviceExternDecl { + export_name: name.to_string(), + param_types: vec![DeviceExternType::pointer_to( + DeviceExternType::Integer(32), + 0, + )], + return_type: DeviceExternType::Void, + attrs: DeviceExternAttrs { + is_convergent, + ..DeviceExternAttrs::default() + }, + }); + } + ReturnOp::new(ctx, None) + .get_operation() + .insert_at_back(entry, ctx); + caller.get_operation().insert_at_back(module_block, ctx); + (module, externs) +} + +#[test] +fn device_extern_convergence_is_per_declaration_in_both_dialects() { + let mut ctx = Context::new(); + let (module, mut externs) = mixed_convergence_module(&mut ctx); + // Repeated identical declarations are still emitted exactly once. + externs.push(externs[0].clone()); + for (dialect, pointer) in [ + (NvvmIrDialect::Modern, "ptr"), + (NvvmIrDialect::LegacyLlvm7, "i32*"), + ] { + let ir = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::new(dialect)) + .unwrap(); + let declarations: Vec<_> = ir + .lines() + .filter(|line| line.starts_with("declare ")) + .collect(); + assert_eq!( + declarations, + [ + format!("declare void @collective({pointer}) convergent"), + format!("declare void @ordinary({pointer})"), + ], + "{ir}" + ); + assert!( + ir.lines().any(|line| { + line.contains(&format!("call void @collective({pointer} ")) && line.ends_with(" #0") + }), + "{ir}" + ); + if dialect == NvvmIrDialect::LegacyLlvm7 { + assert!(ir.contains("bitcast i8* %v0 to i32*"), "{ir}"); + } + } + + externs[2].attrs.is_convergent = false; + let error = export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .expect_err("conflicting convergence requirements must fail"); + assert!( + error.contains("conflicting device-extern declarations for `@collective`: convergent is true versus false"), + "{error}" + ); +} + +#[test] +#[ignore = "requires an installed CUDA Toolkit with modern and legacy libNVVM"] +fn device_extern_convergence_compiles_with_libnvvm_in_both_dialects() { + let nvvm = libnvvm_sys::LibNvvm::load().expect("CUDA Toolkit with libNVVM required"); + let mut ctx = Context::new(); + let (module, externs) = mixed_convergence_module(&mut ctx); + for (dialect, arch) in [ + (NvvmIrDialect::Modern, "-arch=compute_120"), + (NvvmIrDialect::LegacyLlvm7, "-arch=compute_80"), + ] { + let ir = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::new(dialect)) + .unwrap(); + let mut program = libnvvm_sys::Program::new(&nvvm).unwrap(); + program.add_module(ir.as_bytes(), "convergence.ll").unwrap(); + let lto = program.compile(&[arch, "-gen-lto"]).unwrap(); + assert!(!lto.is_empty()); + } +} + #[test] fn legacy_device_extern_adapts_exact_pointer_arguments_and_results() { let mut ctx = Context::new(); diff --git a/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs b/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs index 010bc715d2..e97f6754ab 100644 --- a/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs +++ b/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs @@ -143,6 +143,11 @@ pub const LEGACY_DEVICE_PREFIX: &str = "cuda_oxide_device_246e25db_"; /// linker resolves against the original (unprefixed) external symbol. pub const DEVICE_EXTERN_PREFIX: &str = "cuda_oxide_device_extern_246e25db_"; +/// Doc-alias marker retained after `#[convergent]` macro expansion. +/// Unlike doc comments, aliases survive dependency metadata even for private +/// externs reached through a downstream generic instantiation. +pub const DEVICE_EXTERN_CONVERGENT_MARKER: &str = "cuda_oxide_device_extern_convergent_246e25db"; + /// Prefix added to closure-monomorphization helper functions generated /// by `#[kernel]` for kernels that take a closure parameter. /// diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock index a158e03b8d..5d7825c3c6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock @@ -195,6 +195,13 @@ dependencies = [ name = "cuda-target-spec" version = "0.1.0" +[[package]] +name = "device-ffi-collectives" +version = "0.1.0" +dependencies = [ + "cuda-device", +] + [[package]] name = "device_ffi_test" version = "0.1.0" @@ -202,6 +209,7 @@ dependencies = [ "cuda-core", "cuda-device", "cuda-host", + "device-ffi-collectives", ] [[package]] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml index 28ace2a1d4..3c7fef7608 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml @@ -11,3 +11,4 @@ license = "Apache-2.0" cuda-device = { path = "../../../cuda-device" } cuda-host = { path = "../../../cuda-host" } cuda-core = { path = "../../../../../cuda-core" } +device-ffi-collectives = { path = "convergence-helper" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md index d02e6056cd..b71a620120 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md @@ -125,16 +125,17 @@ device_ffi_test/ ## How It Works -### 1. Rust Kernel Code (`src/main.rs`) +### 1. Rust Kernel Code (simplified) ```rust -use cuda_device::{device, kernel}; +use cuda_device::{convergent, device, kernel}; // Declare external device functions from CUDA C++ -// Note: No NVVM attributes needed - see "Why No NVVM Attributes?" below +// Mark collectives before the Rust device module is optimized. #[device] unsafe extern "C" { fn magnitude_squared(x: f32, y: f32) -> f32; + #[convergent] fn warp_reduce_sum(val: f32) -> f32; } @@ -205,46 +206,27 @@ callers that deliberately use this mapping may allow that lint on the extern. --- -## Why No NVVM Attributes on Extern Declarations? +## Convergence on Extern Declarations -When using LTOIR linking, **NVVM attributes (`#[convergent]`, `#[pure]`, `#[readonly]`) -are not needed on Rust extern function declarations**. Here's why: +Mark external warp and block collective declarations with `#[convergent]`. +cuda-oxide emits LLVM's `convergent` attribute on each marked declaration before +libNVVM optimizes the Rust device module. Attributes on external definitions +arrive later during LTO and cannot protect earlier optimization. -### The LTOIR Already Has Proper Attributes +The annotation forbids transformations that add control dependencies to the +call. It does not make threads reach the call or guarantee reconvergence; +callers must satisfy the collective's warp or block participation requirements. +Reachable declarations of the same symbol must agree on the annotation. Ordinary +scalar helpers and storage-size queries do not need this annotation. -External CUDA C++ compiled with `nvcc -dc -dlto` produces LTOIR with complete -attribute specifications on the function **definitions**: +The `convergence-helper` dependency declares `warp_reduce_sum` privately and +calls it through a public generic device helper. `verify-code-shape.sh` checks +that this declaration retains convergence in the consuming module, alongside +local collective declarations and unmarked scalar declarations. -```llvm -; From cccl_wrappers_text.ltoir -attributes #0 = { convergent inlinehint mustprogress willreturn } -attributes #7 = { convergent nounwind } -``` - -### nvJitLink Uses Definition Attributes - -During LTO linking: -1. cuda-oxide LTOIR has **declarations** (no function body) -2. External LTOIR has **definitions** with proper attributes -3. nvJitLink resolves symbols and uses the **definition's attributes** for optimization - -### Attributes on Declarations Are Redundant - -Since the linker uses the definition's attributes: -- Attributes on declarations don't affect the final code -- The external library authors already set correct attributes -- User-specified attributes could be incorrect or misleading - -### What About cuda-oxide Built-in Intrinsics? - -For cuda-oxide's own intrinsics (like `sync_threads()`, `shuffle()`, etc.), the -compiler handles convergent semantics internally. Users don't need to annotate these. - -### Historical Note - -Earlier versions of this example used `#[convergent]`, `#[pure]`, and `#[readonly]` -attributes. These have been removed as they provided no benefit with LTOIR linking -and could mislead users into thinking they were necessary. +Built-in intrinsics such as `sync_threads()` and warp shuffles already carry +convergence information. `#[pure]` and `#[readonly]` do not currently emit LLVM +memory-effect attributes. --- diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml new file mode 100644 index 0000000000..a5a148e40d --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "device-ffi-collectives" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +cuda-device = { path = "../../../../cuda-device" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs new file mode 100644 index 0000000000..52c17256c0 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#[cuda_device::device] +unsafe extern "C" { + // Deliberately private: the downstream generic instantiation must recover + // convergence from dependency metadata without an exported declaration. + #[cuda_device::convergent] + fn warp_reduce_sum(value: f32) -> f32; +} + +/// Reduce a warp and scale its sum in a downstream device instantiation. +/// +/// # Safety +/// +/// All lanes in a full warp must participate in the reduction. +#[cuda_device::device] +pub unsafe fn scaled_warp_sum(value: f32) -> f32 { + unsafe { warp_reduce_sum(value) * SCALE as f32 } +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs index 0deb545121..02b51e9cdb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs @@ -20,16 +20,15 @@ //! cargo run --release //! ``` -use cuda_device::{device, gpu_printf, kernel, shared::DynamicSharedArray}; +use cuda_device::{convergent, device, gpu_printf, kernel, shared::DynamicSharedArray}; use cuda_host::cuda_module; +use device_ffi_collectives::scaled_warp_sum; // ============================================================================= // External device functions from external_device_funcs.cu // -// NOTE: No NVVM attributes (#[convergent], #[pure], #[readonly]) are needed here. -// The external LTOIR already contains proper attributes on the function definitions. -// When nvJitLink performs LTO linking, it uses the definition's attributes. -// See README.md "Why No NVVM Attributes?" for details. +// Collective declarations carry convergence before libNVVM optimization. +// See README.md "Convergence on Extern Declarations" for details. // ============================================================================= #[device] @@ -37,7 +36,7 @@ unsafe extern "C" { fn magnitude_squared(x: f32, y: f32) -> f32; fn fast_rsqrt(x: f32) -> f32; fn dot_product(a: *const f32, b: *const f32, n: i32) -> f32; - fn warp_reduce_sum(val: f32) -> f32; + #[convergent] fn warp_ballot(predicate: i32) -> u32; fn simple_add(a: f32, b: f32) -> f32; // `char` is a plain 32-bit device-extern slot. Rust still warns because C @@ -56,13 +55,16 @@ unsafe extern "C" { // CCCL (CUB) wrapper functions from cccl_wrappers.cu // // These wrap CUB template functions as extern "C" for FFI. -// The CCCL LTOIR already has proper convergent/nounwind attributes. +// Mark collectives so their participation requirements survive prelink optimization. // ============================================================================= #[device] unsafe extern "C" { + #[convergent] fn cub_block_reduce_sum_f32_256(input: f32, temp: *mut u8) -> f32; + #[convergent] fn cub_block_reduce_max_f32_256(input: f32, temp: *mut u8) -> f32; + #[convergent] fn cub_warp_reduce_sum_f32(input: f32) -> f32; fn cub_get_block_reduce_temp_size_256() -> i32; } @@ -88,7 +90,7 @@ mod kernels { let added = unsafe { simple_add(mag, 1.0) }; // Test convergent function (all threads participate) - let sum = unsafe { warp_reduce_sum(added) }; + let sum = unsafe { scaled_warp_sum::<1>(added) }; // Lane 0 writes result if tid.is_multiple_of(32) { diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh index 66bd4fcaf1..e61bf5750d 100755 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh @@ -4,12 +4,14 @@ # A runtime q -> Q check cannot distinguish a plain i32 ABI from the plausible # but wrong `zeroext i32` mapping: both reach the same PTX and hardware result. -# Pin the declaration and calls in the emitted LLVM IR before LTO. +# Pin the declaration and calls in the emitted LLVM IR before LTO. Also check +# that a private extern reached through a dependency's generic helper retains +# convergence, while the ordinary char declarations remain unmarked. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ll="${root}/device_ffi_test.ll" +ll="${1:-${root}/device_ffi_test.ll}" test -s "${ll}" @@ -46,4 +48,19 @@ if grep -Eq '\b(signext|zeroext)\b' <<<"${char_lines}"; then exit 1 fi -echo "device_ffi_test char ABI shape: PASS" +# Calls retain the existing conservative convergence group; only these ordinary +# declarations must be unmarked. The anchored declaration patterns above also +# reject an unexpected attribute trailer. +if grep -E '^declare .*@(char_to_upper|char_store)\(' "${ll}" | grep -qw convergent; then + echo "error: ordinary char declarations must not be convergent" >&2 + exit 1 +fi + +for symbol in warp_reduce_sum cub_warp_reduce_sum_f32; do + require_count "convergent ${symbol} declaration" 1 \ + "^declare float @${symbol}\(float\) convergent$" +done +require_count "convergent warp_ballot declaration" 1 \ + '^declare i32 @warp_ballot\(i32\) convergent$' + +echo "device_ffi_test char ABI and convergence shape: PASS (${ll})" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs index 97e5b7c45a..0b7d2d8ca7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs @@ -354,27 +354,20 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on our -/// declarations redundant. -/// -/// The `#[convergent]`, `#[pure]`, and `#[readonly]` Rust attributes are still parsed -/// but their values are not used in code generation. This struct is retained for -/// potential future use or debugging. +/// Convergence is emitted on the declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive; they are not inferred from external +/// definitions that become available only during LTO. #[derive(Debug, Clone, Default)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). - /// NOTE: Not currently emitted - external LTOIR has proper convergent attrs. + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, /// Function is pure (no side effects, result depends only on inputs). - /// NOTE: Not currently emitted - external LTOIR has proper memory attrs. + /// Inactive for Rust source annotations. pub is_pure: bool, /// Function is read-only (only reads memory, doesn't write). - /// NOTE: Not currently emitted - external LTOIR has proper memory attrs. + /// Inactive for Rust source annotations. pub is_readonly: bool, } @@ -1509,7 +1502,12 @@ impl<'tcx> DeviceCollector<'tcx> { .next() .is_some() }; - attrs.is_convergent = check("convergent"); + attrs.is_convergent = rustc_attr_ir::find_attr!( + self.tcx, def_id, Doc(doc) + if doc.aliases.contains_key(&Symbol::intern( + reserved_oxide_symbols::DEVICE_EXTERN_CONVERGENT_MARKER + )) + ); attrs.is_pure = check("pure"); attrs.is_readonly = check("readonly");