diff --git a/cuda-oxide/crates/cuda-macros/src/lib.rs b/cuda-oxide/crates/cuda-macros/src/lib.rs index 0a3d738b55..75afea241f 100644 --- a/cuda-oxide/crates/cuda-macros/src/lib.rs +++ b/cuda-oxide/crates/cuda-macros/src/lib.rs @@ -1015,10 +1015,14 @@ pub fn tile_export(attr: TokenStream, item: TokenStream) -> TokenStream { // NVVM Attributes for Device FFI // ============================================================================ -/// Marks a device function as convergent. +/// Marks an external device function declaration as convergent. /// -/// Convergent functions must be called by all threads in a warp/block together. -/// This prevents the optimizer from moving calls across control flow boundaries. +/// LLVM forbids transformations that add control dependencies to the call, +/// including before external device definitions are linked. The annotation +/// does not make threads reach the call or guarantee reconvergence. Callers +/// must satisfy the function's warp or block participation requirements. +/// Every declaration of the same symbol reachable in a device module must +/// agree on this annotation; conflicting declarations are rejected. /// /// # When to Use /// @@ -1039,77 +1043,32 @@ pub fn tile_export(attr: TokenStream, item: TokenStream) -> TokenStream { /// # Generated LLVM IR /// /// ```llvm -/// declare float @cub_block_reduce_sum(float, ptr) #0 -/// attributes #0 = { convergent nounwind } +/// declare float @cub_block_reduce_sum(float, ptr) convergent /// ``` #[proc_macro_attribute] pub fn convergent(_attr: TokenStream, item: TokenStream) -> TokenStream { - // This is a marker attribute - just pass through the item unchanged. - // The collector will read this attribute and apply the LLVM convergent attribute. - item + // The procedural attribute disappears during expansion. A doc alias is + // retained in metadata even for private foreign items, unlike doc comments. + let marker = reserved_oxide_symbols::DEVICE_EXTERN_CONVERGENT_MARKER; + let item = proc_macro2::TokenStream::from(item); + quote::quote!(#[doc(alias = #marker)] #item).into() } -/// Marks a device function as pure (no side effects). -/// -/// Pure functions only depend on their inputs and have no side effects. -/// This enables aggressive optimizations like CSE and dead code elimination. -/// -/// # When to Use -/// -/// - Math functions that don't access memory -/// - Functions that compute results purely from input arguments +/// Compatibility annotation for a pure device function. /// -/// # Example -/// -/// ```ignore -/// #[device] -/// extern "C" { -/// #[pure] -/// fn fast_rsqrt(x: f32) -> f32; -/// } -/// ``` -/// -/// # Generated LLVM IR -/// -/// ```llvm -/// declare float @fast_rsqrt(float) #0 -/// attributes #0 = { nounwind readnone } -/// ``` +/// This annotation is currently inactive: it records no compiler effect and +/// emits no LLVM memory-effect attributes. #[proc_macro_attribute] pub fn pure(_attr: TokenStream, item: TokenStream) -> TokenStream { - // Marker attribute - collector will read and apply LLVM readnone attribute item } -/// Marks a device function as read-only. +/// Compatibility annotation for a read-only device function. /// -/// Read-only functions may read memory but never write to it. -/// This enables optimizations like load hoisting and caching. -/// -/// # When to Use -/// -/// - Lookup table functions -/// - Functions that only read from input arrays -/// -/// # Example -/// -/// ```ignore -/// #[device] -/// extern "C" { -/// #[readonly] -/// fn lookup_table(table: *const f32, idx: i32) -> f32; -/// } -/// ``` -/// -/// # Generated LLVM IR -/// -/// ```llvm -/// declare float @lookup_table(ptr, i32) #0 -/// attributes #0 = { nounwind readonly } -/// ``` +/// This annotation is currently inactive: it records no compiler effect and +/// emits no LLVM memory-effect attributes. #[proc_macro_attribute] pub fn readonly(_attr: TokenStream, item: TokenStream) -> TokenStream { - // Marker attribute - collector will read and apply LLVM readonly attribute item } diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs index 12fe5ecda2..fc1b5521db 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs @@ -44,25 +44,19 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on our -/// declarations redundant. -/// -/// This struct is retained for the pipeline API but values are not used in code generation. +/// Convergence is emitted on each marked declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). - /// NOTE: Not currently emitted to LLVM IR. + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, /// Function is pure (no side effects, result depends only on inputs). - /// NOTE: Not currently emitted to LLVM IR. + /// Inactive; not emitted to LLVM IR. pub is_pure: bool, /// Function is read-only (only reads memory, doesn't write). - /// NOTE: Not currently emitted to LLVM IR. + /// Inactive; not emitted to LLVM IR. pub is_readonly: bool, } diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs index 05011e5e58..d08f68f8d5 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs @@ -84,7 +84,7 @@ pub fn add_device_extern_declarations( region.iter(ctx).next().expect("Module should have a block") }; - let declared_symbols: HashMap<_, _> = block + let mut declared_symbols: HashMap<_, _> = block .deref(ctx) .iter(ctx) .filter_map(|op| { @@ -140,6 +140,7 @@ pub fn add_device_extern_declarations( // Insert at the front of the module (declarations come before definitions) func_op.get_operation().insert_at_front(block, ctx); + declared_symbols.insert(decl.export_name.clone(), func_type); } Ok(()) @@ -196,3 +197,62 @@ fn device_extern_type_to_pliron( } }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::export::DeviceExternAttrs; + use llvm_export::export::{NvvmExportConfig, export_module_with_externs}; + use pliron::builtin::ops::ModuleOp; + use pliron::common_traits::Verify; + + #[test] + fn repeated_externs_keep_one_operation_and_preserve_conflict_checks() { + let mut ctx = Context::new(); + let module = ModuleOp::new(&mut ctx, "externs".try_into().unwrap()); + let declaration = DeviceExternDecl { + export_name: "external".to_string(), + param_types: vec![DeviceExternType::Integer(32)], + return_type: DeviceExternType::Void, + attrs: DeviceExternAttrs::default(), + }; + let mut externs = [declaration.clone(), declaration]; + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap(); + let declarations = { + let region = module.get_operation().deref(&ctx).get_region(0).deref(&ctx); + let block = region.iter(&ctx).next().unwrap(); + block + .deref(&ctx) + .iter(&ctx) + .filter(|&op| { + Operation::get_op::(op, &ctx).is_some_and( + |function| function.get_symbol_name(&ctx).to_string() == "external", + ) + }) + .count() + }; + assert_eq!(declarations, 1); + module.get_operation().deref(&ctx).verify(&ctx).unwrap(); + let ir = export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .unwrap(); + assert_eq!(ir.matches("declare void @external(i32)").count(), 1); + + // Lowering shares only the FuncOp. The complete declarations still + // reach export, where differing convergence requirements are rejected. + externs[1].attrs.is_convergent = true; + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap(); + let error = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .unwrap_err(); + assert!(error.contains("convergent is false versus true"), "{error}"); + + externs[1].param_types[0] = DeviceExternType::Float32; + let error = + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap_err(); + assert!( + error + .to_string() + .contains("conflicts with the call-site declaration") + ); + } +} diff --git a/cuda-oxide/crates/llvm-export/src/export/externs.rs b/cuda-oxide/crates/llvm-export/src/export/externs.rs index ceeab494f0..2cea5a5fc0 100644 --- a/cuda-oxide/crates/llvm-export/src/export/externs.rs +++ b/cuda-oxide/crates/llvm-export/src/export/externs.rs @@ -225,22 +225,18 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on -/// declarations redundant. -/// -/// This struct is retained for potential future use or for debugging/inspection. +/// Convergence is emitted on the declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive; they are not inferred from external +/// definitions that become available only during LTO. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, - /// Function is pure (no side effects). Maps to LLVM `readnone`. + /// Function is pure (no side effects). Inactive for Rust source annotations. pub is_pure: bool, - /// Function is read-only (only reads memory). Maps to LLVM `readonly`. + /// Function is read-only (only reads memory). Inactive for Rust source annotations. pub is_readonly: bool, } diff --git a/cuda-oxide/crates/llvm-export/src/export/module.rs b/cuda-oxide/crates/llvm-export/src/export/module.rs index 1a9e934851..9d425d54c4 100644 --- a/cuda-oxide/crates/llvm-export/src/export/module.rs +++ b/cuda-oxide/crates/llvm-export/src/export/module.rs @@ -487,10 +487,17 @@ fn index_device_externs( validate_device_extern_function_shape(state, decl)?; if let Some(previous) = state.device_externs.get(&decl.export_name) { if previous != decl { - return Err(format!( + let conflict = format!( "conflicting device-extern declarations for `@{}`", decl.export_name - )); + ); + if previous.attrs.is_convergent != decl.attrs.is_convergent { + return Err(format!( + "{conflict}: convergent is {} versus {}", + previous.attrs.is_convergent, decl.attrs.is_convergent + )); + } + return Err(conflict); } continue; } @@ -610,12 +617,9 @@ pub(super) fn export_module_with_externs_impl( // 2. Device extern declarations (before function definitions) // - // NOTE: We intentionally do NOT emit LLVM attributes on these declarations. - // The external LTOIR (from nvcc -dc -dlto) already contains proper attributes - // (convergent, nounwind, memory, etc.) on the function DEFINITIONS. - // When nvJitLink performs LTO linking, it uses the definition's attributes. - // Attributes on declarations are redundant and were causing issues where - // all externs incorrectly got the same attribute group. + // Convergence must be visible before libNVVM optimizes this module, not + // only when external definitions arrive at LTO. Emit it per declaration + // so unrelated externs do not inherit a shared attribute group. if !device_externs.is_empty() { writeln!( &mut output, @@ -643,7 +647,11 @@ pub(super) fn export_module_with_externs_impl( } param.write_llvm_with_attr(&mut output, state.legacy_typed_pointers())?; } - writeln!(&mut output, ")").unwrap(); + write!(&mut output, ")").unwrap(); + if decl.attrs.is_convergent { + write!(&mut output, " convergent").unwrap(); + } + writeln!(&mut output).unwrap(); } writeln!(&mut output).unwrap(); } @@ -711,7 +719,7 @@ pub(super) fn export_module_with_externs_impl( } // 6. Emit attribute groups for convergent intrinsics used by module functions - // Note: Device extern declarations no longer get attribute groups - see section 2 comment. + // Device extern convergence is emitted inline, independently of this group. if state.convergent_used { writeln!(&mut output).unwrap(); writeln!(&mut output, "attributes #0 = {{ convergent }}").unwrap(); diff --git a/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs b/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs index ee928c3819..f68a227611 100644 --- a/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs +++ b/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs @@ -23,6 +23,113 @@ use pliron::{ use crate::common::module_top_block; +fn mixed_convergence_module(ctx: &mut Context) -> (ModuleOp, Vec) { + let module = ModuleOp::new(ctx, "mixed_convergence".try_into().unwrap()); + let module_block = module_top_block(ctx, &module); + let ptr_ty = PointerType::get(ctx, 0); + let void_ty = VoidType::get(ctx); + let external_ty = FuncType::get(ctx, void_ty.into(), vec![ptr_ty.into()], false); + let caller = FuncOp::new(ctx, "caller".try_into().unwrap(), external_ty); + let entry = caller.get_or_create_entry_block(ctx); + let pointer = entry.deref(ctx).get_argument(0); + let mut externs = Vec::new(); + for (name, is_convergent) in [("collective", true), ("ordinary", false)] { + FuncOp::new(ctx, name.try_into().unwrap(), external_ty) + .get_operation() + .insert_at_back(module_block, ctx); + CallOp::new( + ctx, + CallOpCallable::Direct(name.try_into().unwrap()), + external_ty, + vec![pointer], + ) + .get_operation() + .insert_at_back(entry, ctx); + externs.push(DeviceExternDecl { + export_name: name.to_string(), + param_types: vec![DeviceExternType::pointer_to( + DeviceExternType::Integer(32), + 0, + )], + return_type: DeviceExternType::Void, + attrs: DeviceExternAttrs { + is_convergent, + ..DeviceExternAttrs::default() + }, + }); + } + ReturnOp::new(ctx, None) + .get_operation() + .insert_at_back(entry, ctx); + caller.get_operation().insert_at_back(module_block, ctx); + (module, externs) +} + +#[test] +fn device_extern_convergence_is_per_declaration_in_both_dialects() { + let mut ctx = Context::new(); + let (module, mut externs) = mixed_convergence_module(&mut ctx); + // Repeated identical declarations are still emitted exactly once. + externs.push(externs[0].clone()); + for (dialect, pointer) in [ + (NvvmIrDialect::Modern, "ptr"), + (NvvmIrDialect::LegacyLlvm7, "i32*"), + ] { + let ir = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::new(dialect)) + .unwrap(); + let declarations: Vec<_> = ir + .lines() + .filter(|line| line.starts_with("declare ")) + .collect(); + assert_eq!( + declarations, + [ + format!("declare void @collective({pointer}) convergent"), + format!("declare void @ordinary({pointer})"), + ], + "{ir}" + ); + assert!( + ir.lines().any(|line| { + line.contains(&format!("call void @collective({pointer} ")) && line.ends_with(" #0") + }), + "{ir}" + ); + if dialect == NvvmIrDialect::LegacyLlvm7 { + assert!(ir.contains("bitcast i8* %v0 to i32*"), "{ir}"); + } + } + + externs[2].attrs.is_convergent = false; + let error = export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .expect_err("conflicting convergence requirements must fail"); + assert!( + error.contains("conflicting device-extern declarations for `@collective`: convergent is true versus false"), + "{error}" + ); +} + +#[test] +#[ignore = "requires an installed CUDA Toolkit with modern and legacy libNVVM"] +fn device_extern_convergence_compiles_with_libnvvm_in_both_dialects() { + let nvvm = libnvvm_sys::LibNvvm::load().expect("CUDA Toolkit with libNVVM required"); + let mut ctx = Context::new(); + let (module, externs) = mixed_convergence_module(&mut ctx); + for (dialect, arch) in [ + (NvvmIrDialect::Modern, "-arch=compute_120"), + (NvvmIrDialect::LegacyLlvm7, "-arch=compute_80"), + ] { + let ir = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::new(dialect)) + .unwrap(); + let mut program = libnvvm_sys::Program::new(&nvvm).unwrap(); + program.add_module(ir.as_bytes(), "convergence.ll").unwrap(); + let lto = program.compile(&[arch, "-gen-lto"]).unwrap(); + assert!(!lto.is_empty()); + } +} + #[test] fn legacy_device_extern_adapts_exact_pointer_arguments_and_results() { let mut ctx = Context::new(); diff --git a/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs b/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs index 010bc715d2..e97f6754ab 100644 --- a/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs +++ b/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs @@ -143,6 +143,11 @@ pub const LEGACY_DEVICE_PREFIX: &str = "cuda_oxide_device_246e25db_"; /// linker resolves against the original (unprefixed) external symbol. pub const DEVICE_EXTERN_PREFIX: &str = "cuda_oxide_device_extern_246e25db_"; +/// Doc-alias marker retained after `#[convergent]` macro expansion. +/// Unlike doc comments, aliases survive dependency metadata even for private +/// externs reached through a downstream generic instantiation. +pub const DEVICE_EXTERN_CONVERGENT_MARKER: &str = "cuda_oxide_device_extern_convergent_246e25db"; + /// Prefix added to closure-monomorphization helper functions generated /// by `#[kernel]` for kernels that take a closure parameter. /// diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock index a158e03b8d..5d7825c3c6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock @@ -195,6 +195,13 @@ dependencies = [ name = "cuda-target-spec" version = "0.1.0" +[[package]] +name = "device-ffi-collectives" +version = "0.1.0" +dependencies = [ + "cuda-device", +] + [[package]] name = "device_ffi_test" version = "0.1.0" @@ -202,6 +209,7 @@ dependencies = [ "cuda-core", "cuda-device", "cuda-host", + "device-ffi-collectives", ] [[package]] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml index 28ace2a1d4..3c7fef7608 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml @@ -11,3 +11,4 @@ license = "Apache-2.0" cuda-device = { path = "../../../cuda-device" } cuda-host = { path = "../../../cuda-host" } cuda-core = { path = "../../../../../cuda-core" } +device-ffi-collectives = { path = "convergence-helper" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md index d02e6056cd..b71a620120 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md @@ -125,16 +125,17 @@ device_ffi_test/ ## How It Works -### 1. Rust Kernel Code (`src/main.rs`) +### 1. Rust Kernel Code (simplified) ```rust -use cuda_device::{device, kernel}; +use cuda_device::{convergent, device, kernel}; // Declare external device functions from CUDA C++ -// Note: No NVVM attributes needed - see "Why No NVVM Attributes?" below +// Mark collectives before the Rust device module is optimized. #[device] unsafe extern "C" { fn magnitude_squared(x: f32, y: f32) -> f32; + #[convergent] fn warp_reduce_sum(val: f32) -> f32; } @@ -205,46 +206,27 @@ callers that deliberately use this mapping may allow that lint on the extern. --- -## Why No NVVM Attributes on Extern Declarations? +## Convergence on Extern Declarations -When using LTOIR linking, **NVVM attributes (`#[convergent]`, `#[pure]`, `#[readonly]`) -are not needed on Rust extern function declarations**. Here's why: +Mark external warp and block collective declarations with `#[convergent]`. +cuda-oxide emits LLVM's `convergent` attribute on each marked declaration before +libNVVM optimizes the Rust device module. Attributes on external definitions +arrive later during LTO and cannot protect earlier optimization. -### The LTOIR Already Has Proper Attributes +The annotation forbids transformations that add control dependencies to the +call. It does not make threads reach the call or guarantee reconvergence; +callers must satisfy the collective's warp or block participation requirements. +Reachable declarations of the same symbol must agree on the annotation. Ordinary +scalar helpers and storage-size queries do not need this annotation. -External CUDA C++ compiled with `nvcc -dc -dlto` produces LTOIR with complete -attribute specifications on the function **definitions**: +The `convergence-helper` dependency declares `warp_reduce_sum` privately and +calls it through a public generic device helper. `verify-code-shape.sh` checks +that this declaration retains convergence in the consuming module, alongside +local collective declarations and unmarked scalar declarations. -```llvm -; From cccl_wrappers_text.ltoir -attributes #0 = { convergent inlinehint mustprogress willreturn } -attributes #7 = { convergent nounwind } -``` - -### nvJitLink Uses Definition Attributes - -During LTO linking: -1. cuda-oxide LTOIR has **declarations** (no function body) -2. External LTOIR has **definitions** with proper attributes -3. nvJitLink resolves symbols and uses the **definition's attributes** for optimization - -### Attributes on Declarations Are Redundant - -Since the linker uses the definition's attributes: -- Attributes on declarations don't affect the final code -- The external library authors already set correct attributes -- User-specified attributes could be incorrect or misleading - -### What About cuda-oxide Built-in Intrinsics? - -For cuda-oxide's own intrinsics (like `sync_threads()`, `shuffle()`, etc.), the -compiler handles convergent semantics internally. Users don't need to annotate these. - -### Historical Note - -Earlier versions of this example used `#[convergent]`, `#[pure]`, and `#[readonly]` -attributes. These have been removed as they provided no benefit with LTOIR linking -and could mislead users into thinking they were necessary. +Built-in intrinsics such as `sync_threads()` and warp shuffles already carry +convergence information. `#[pure]` and `#[readonly]` do not currently emit LLVM +memory-effect attributes. --- diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml new file mode 100644 index 0000000000..a5a148e40d --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "device-ffi-collectives" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +cuda-device = { path = "../../../../cuda-device" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs new file mode 100644 index 0000000000..52c17256c0 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#[cuda_device::device] +unsafe extern "C" { + // Deliberately private: the downstream generic instantiation must recover + // convergence from dependency metadata without an exported declaration. + #[cuda_device::convergent] + fn warp_reduce_sum(value: f32) -> f32; +} + +/// Reduce a warp and scale its sum in a downstream device instantiation. +/// +/// # Safety +/// +/// All lanes in a full warp must participate in the reduction. +#[cuda_device::device] +pub unsafe fn scaled_warp_sum(value: f32) -> f32 { + unsafe { warp_reduce_sum(value) * SCALE as f32 } +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs index 0deb545121..02b51e9cdb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs @@ -20,16 +20,15 @@ //! cargo run --release //! ``` -use cuda_device::{device, gpu_printf, kernel, shared::DynamicSharedArray}; +use cuda_device::{convergent, device, gpu_printf, kernel, shared::DynamicSharedArray}; use cuda_host::cuda_module; +use device_ffi_collectives::scaled_warp_sum; // ============================================================================= // External device functions from external_device_funcs.cu // -// NOTE: No NVVM attributes (#[convergent], #[pure], #[readonly]) are needed here. -// The external LTOIR already contains proper attributes on the function definitions. -// When nvJitLink performs LTO linking, it uses the definition's attributes. -// See README.md "Why No NVVM Attributes?" for details. +// Collective declarations carry convergence before libNVVM optimization. +// See README.md "Convergence on Extern Declarations" for details. // ============================================================================= #[device] @@ -37,7 +36,7 @@ unsafe extern "C" { fn magnitude_squared(x: f32, y: f32) -> f32; fn fast_rsqrt(x: f32) -> f32; fn dot_product(a: *const f32, b: *const f32, n: i32) -> f32; - fn warp_reduce_sum(val: f32) -> f32; + #[convergent] fn warp_ballot(predicate: i32) -> u32; fn simple_add(a: f32, b: f32) -> f32; // `char` is a plain 32-bit device-extern slot. Rust still warns because C @@ -56,13 +55,16 @@ unsafe extern "C" { // CCCL (CUB) wrapper functions from cccl_wrappers.cu // // These wrap CUB template functions as extern "C" for FFI. -// The CCCL LTOIR already has proper convergent/nounwind attributes. +// Mark collectives so their participation requirements survive prelink optimization. // ============================================================================= #[device] unsafe extern "C" { + #[convergent] fn cub_block_reduce_sum_f32_256(input: f32, temp: *mut u8) -> f32; + #[convergent] fn cub_block_reduce_max_f32_256(input: f32, temp: *mut u8) -> f32; + #[convergent] fn cub_warp_reduce_sum_f32(input: f32) -> f32; fn cub_get_block_reduce_temp_size_256() -> i32; } @@ -88,7 +90,7 @@ mod kernels { let added = unsafe { simple_add(mag, 1.0) }; // Test convergent function (all threads participate) - let sum = unsafe { warp_reduce_sum(added) }; + let sum = unsafe { scaled_warp_sum::<1>(added) }; // Lane 0 writes result if tid.is_multiple_of(32) { diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh index 66bd4fcaf1..e61bf5750d 100755 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh @@ -4,12 +4,14 @@ # A runtime q -> Q check cannot distinguish a plain i32 ABI from the plausible # but wrong `zeroext i32` mapping: both reach the same PTX and hardware result. -# Pin the declaration and calls in the emitted LLVM IR before LTO. +# Pin the declaration and calls in the emitted LLVM IR before LTO. Also check +# that a private extern reached through a dependency's generic helper retains +# convergence, while the ordinary char declarations remain unmarked. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ll="${root}/device_ffi_test.ll" +ll="${1:-${root}/device_ffi_test.ll}" test -s "${ll}" @@ -46,4 +48,19 @@ if grep -Eq '\b(signext|zeroext)\b' <<<"${char_lines}"; then exit 1 fi -echo "device_ffi_test char ABI shape: PASS" +# Calls retain the existing conservative convergence group; only these ordinary +# declarations must be unmarked. The anchored declaration patterns above also +# reject an unexpected attribute trailer. +if grep -E '^declare .*@(char_to_upper|char_store)\(' "${ll}" | grep -qw convergent; then + echo "error: ordinary char declarations must not be convergent" >&2 + exit 1 +fi + +for symbol in warp_reduce_sum cub_warp_reduce_sum_f32; do + require_count "convergent ${symbol} declaration" 1 \ + "^declare float @${symbol}\(float\) convergent$" +done +require_count "convergent warp_ballot declaration" 1 \ + '^declare i32 @warp_ballot\(i32\) convergent$' + +echo "device_ffi_test char ABI and convergence shape: PASS (${ll})" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs index 97e5b7c45a..0b7d2d8ca7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs @@ -354,27 +354,20 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on our -/// declarations redundant. -/// -/// The `#[convergent]`, `#[pure]`, and `#[readonly]` Rust attributes are still parsed -/// but their values are not used in code generation. This struct is retained for -/// potential future use or debugging. +/// Convergence is emitted on the declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive; they are not inferred from external +/// definitions that become available only during LTO. #[derive(Debug, Clone, Default)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). - /// NOTE: Not currently emitted - external LTOIR has proper convergent attrs. + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, /// Function is pure (no side effects, result depends only on inputs). - /// NOTE: Not currently emitted - external LTOIR has proper memory attrs. + /// Inactive for Rust source annotations. pub is_pure: bool, /// Function is read-only (only reads memory, doesn't write). - /// NOTE: Not currently emitted - external LTOIR has proper memory attrs. + /// Inactive for Rust source annotations. pub is_readonly: bool, } @@ -1509,7 +1502,12 @@ impl<'tcx> DeviceCollector<'tcx> { .next() .is_some() }; - attrs.is_convergent = check("convergent"); + attrs.is_convergent = rustc_attr_ir::find_attr!( + self.tcx, def_id, Doc(doc) + if doc.aliases.contains_key(&Symbol::intern( + reserved_oxide_symbols::DEVICE_EXTERN_CONVERGENT_MARKER + )) + ); attrs.is_pure = check("pure"); attrs.is_readonly = check("readonly");