From ffa23dba55aa3412b5ad01e05172bdcb427717f4 Mon Sep 17 00:00:00 2001 From: Yunus Mutlu Date: Mon, 28 Sep 2026 15:19:27 +0300 Subject: [PATCH 1/5] fix(mir-lower): cap enum alignment anchors (refs #1356) Signed-off-by: Yunus Mutlu --- .../src/convert/ops/memory/tests/enums.rs | 62 +++++++++++++++++++ .../src/convert/types/enum_layout.rs | 13 ++-- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs b/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs index 5658a39cc1..0c1c453e09 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs @@ -44,6 +44,68 @@ fn unit_variants(n: usize) -> Vec { (0..n).map(|i| EnumVariant::unit(format!("V{i}"))).collect() } +/// Converting a `repr(align(2^29))` byte alignment directly to an LLVM +/// integer bit width wraps at the `u32` boundary: `2^29 * 8 == 2^32`. +/// Keep the storage type's natural alignment within NVPTX's scalar alignment +/// limit; memory operations carry the stronger Rust ABI alignment explicitly. +#[test] +fn enum_conversion_handles_repr_align_bitwidth_boundary() { + use crate::convert::types::{llvm_type_size_align, mir_type_abi_align}; + + for (align, anchor_bits) in [ + (8u64, 64), + (16, 128), + (32, 128), + (64, 128), + (1 << 20, 128), + (1 << 29, 128), + ] { + let mut ctx = make_ctx(); + let ty = make_enum_ty(&mut ctx, "AlignedEnum", 8, unit_variants(2), align, align); + let conv = convert_type(&mut ctx, ty).unwrap(); + let array_ty: TypeHandle = MirArrayType::get(&mut ctx, ty, 3).into(); + let conv_array = convert_type(&mut ctx, array_ty).unwrap(); + let natural_align = u64::from(anchor_bits / 8); + + assert_eq!( + llvm_type_size_align(&ctx, conv), + Some((align, natural_align)), + "enum with {align}-byte alignment keeps rustc size" + ); + assert_eq!( + llvm_type_size_align(&ctx, conv_array), + Some((align * 3, natural_align)), + "enum array with {align}-byte alignment keeps rustc stride" + ); + assert_eq!(mir_type_abi_align(&ctx, ty), Some(align)); + assert_eq!(mir_type_abi_align(&ctx, array_ty), Some(align)); + + let conv_ref = conv.deref(&ctx); + let struct_ty = conv_ref + .downcast_ref::() + .expect("converted enum is a struct"); + let anchor_ty = struct_ty + .fields() + .next() + .expect("expected alignment anchor"); + let anchor_ref = anchor_ty.deref(&ctx); + let anchor = anchor_ref + .downcast_ref::() + .expect("alignment anchor is an array"); + assert_eq!(anchor.size(), 0, "alignment anchor must occupy no bytes"); + let anchor_int = anchor.elem_type(); + let anchor_int_ref = anchor_int.deref(&ctx); + assert_eq!( + anchor_int_ref + .downcast_ref::() + .expect("alignment anchor element is an integer") + .width(), + anchor_bits, + "enum with {align}-byte alignment uses a bounded integer anchor" + ); + } +} + /// Converted enum allocation size must equal rustc's `total_size` for /// every memory-faithful tag shape: that size is what GEP strides by. #[test] diff --git a/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs b/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs index 25eb043d10..0531dc78e2 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs @@ -507,12 +507,13 @@ pub(crate) fn build_enum_slot_map( )); } if natural_align < required_align { - // The byte claims alone can under-align the storage, e.g. when the - // only claim is an i8 niche carrier inside a 4-aligned enum. Raise - // the struct's alignment with a zero-length anchor field, the same - // mechanism union storage uses; it occupies no bytes, so every slot - // index simply shifts by one. - let anchor_int = IntegerType::get(ctx, (required_align * 8) as u32, Signedness::Signless); + // Raise the storage's natural alignment with a zero-length anchor; + // it occupies no bytes, so the claim slots only shift by one. + // NVPTX scalar alignment tops out at 16 bytes. Cap the anchor before + // converting alignment to bit width; stronger Rust ABI alignment is + // carried explicitly by memory operations. + let anchor_align = required_align.min(16); + let anchor_int = IntegerType::get(ctx, (anchor_align * 8) as u32, Signedness::Signless); let anchor: TypeHandle = llvm_types::ArrayType::get(ctx, anchor_int.into(), 0).into(); llvm_fields.insert(0, anchor); for slot in &mut slot_of_claim { From 289cfdf2f94f843b2bbc9f3be0a5cb1658cb80cb Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:50:35 +0530 Subject: [PATCH 2/5] fix(mir-lower): share the capped alignment anchor between unions and enums Union and enum storage each built their own zero-length alignment anchor and each capped it at a bare 16. One helper, alignment_anchor_type, now builds the anchor for both, and MAX_NATURAL_ALIGN names the cap for the layout model and the test fixtures as well. The helper's doc states the real reason for the cap: the exporter's data layout strings, llvm_type_size_align and the exporter's natural_alignment all stop at i128, while llc's default NVPTX layout has i256:256, so a wider anchor made opt and llc disagree on a type's alignment and raised nothing. Rust alignment is stamped on the memory operations. The three comments that said NVPTX itself stops at 16 bytes now point at the helper. Tests: the enum anchor table gains the 2^21 row (the first width LLVM rejects; 2^29 is the u32 wrap), and a lowering-level regression beside the union one checks that an over-aligned enum array keeps the Rust alignment on the alloca and the store at 32, 64 and 2^29 while the storage keeps rustc's size and stride. Both fail with an unbounded anchor. Signed-off-by: nihalpasham --- .../src/convert/ops/aggregate/test_support.rs | 4 +- .../src/convert/ops/memory/tests/enums.rs | 88 ++++++++++++++++++- .../src/convert/types/enum_layout.rs | 18 ++-- .../mir-lower/src/convert/types/layout.rs | 45 +++++++++- .../crates/mir-lower/src/convert/types/mod.rs | 4 + .../src/convert/types/union_storage.rs | 12 ++- 6 files changed, 148 insertions(+), 23 deletions(-) diff --git a/cuda-oxide/crates/mir-lower/src/convert/ops/aggregate/test_support.rs b/cuda-oxide/crates/mir-lower/src/convert/ops/aggregate/test_support.rs index 22d3a22d4a..bc66f12485 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/ops/aggregate/test_support.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/ops/aggregate/test_support.rs @@ -18,6 +18,8 @@ use pliron::operation::Operation; use pliron::r#type::TypeHandle; use pliron::value::Value; +use crate::convert::types::MAX_NATURAL_ALIGN; + pub(super) fn insert_indices(ctx: &Context, inserts: &[llvm::InsertValueOp]) -> Vec> { inserts.iter().map(|op| op.indices(ctx)).collect() } @@ -82,7 +84,7 @@ pub(super) fn unit_niche_enum( .collect::>(); let discriminants = (0..inhabited.len() as u64).collect(); let carrier_size = u64::from(carrier_width).div_ceil(8); - let carrier_align = carrier_size.next_power_of_two().min(16); + let carrier_align = carrier_size.next_power_of_two().min(MAX_NATURAL_ALIGN); MirEnumType::get_with_encoding( ctx, "UnitNiche".into(), diff --git a/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs b/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs index 0c1c453e09..884c68169a 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs @@ -44,10 +44,15 @@ fn unit_variants(n: usize) -> Vec { (0..n).map(|i| EnumVariant::unit(format!("V{i}"))).collect() } -/// Converting a `repr(align(2^29))` byte alignment directly to an LLVM -/// integer bit width wraps at the `u32` boundary: `2^29 * 8 == 2^32`. -/// Keep the storage type's natural alignment within NVPTX's scalar alignment -/// limit; memory operations carry the stronger Rust ABI alignment explicitly. +/// The alignment anchor is `[0 x iN]` with `N = 8 * align` bits, so an +/// unbounded anchor failed in two ways: from `repr(align(2^21))` up the +/// width passed LLVM's 2^23-bit integer limit (`i8388608` is the last width +/// LLVM accepts, `i8388609` the first it rejects), and at `repr(align(2^29))` +/// the `u32` conversion wrapped to `i0`. `alignment_anchor_type` caps the +/// anchor at `MAX_NATURAL_ALIGN` (16 bytes), which is where cuda-oxide's data +/// layouts and layout model stop anyway; memory operations carry the stronger +/// Rust ABI alignment explicitly. The 8 and 16 rows are the controls, 2^20 is +/// the last width LLVM accepted, 2^21 the first it rejected, 2^29 the wrap. #[test] fn enum_conversion_handles_repr_align_bitwidth_boundary() { use crate::convert::types::{llvm_type_size_align, mir_type_abi_align}; @@ -58,6 +63,7 @@ fn enum_conversion_handles_repr_align_bitwidth_boundary() { (32, 128), (64, 128), (1 << 20, 128), + (1 << 21, 128), (1 << 29, 128), ] { let mut ctx = make_ctx(); @@ -106,6 +112,80 @@ fn enum_conversion_handles_repr_align_bitwidth_boundary() { } } +/// Lowering a reference to an over-aligned enum array keeps the Rust ABI +/// alignment on the alloca and the store, while the storage type keeps +/// rustc's size and stride at the capped 16-byte natural alignment. Enum +/// sibling of `convert_ref_preserves_over_aligned_union_array_layout_and_alignment` +/// in `access.rs`; 2^29 is the alignment whose anchor used to wrap to `i0`. +#[test] +fn convert_ref_preserves_over_aligned_enum_array_layout_and_alignment() { + for abi_align in [32u64, 64, 1 << 29] { + let mut ctx = make_ctx(); + let enum_ty = make_enum_ty( + &mut ctx, + "AlignedEnum", + 8, + unit_variants(2), + abi_align, + abi_align, + ); + let array_ty: TypeHandle = MirArrayType::get(&mut ctx, enum_ty, 3).into(); + let mir_ptr_ty = + MirPtrType::get_generic_with_kind(&mut ctx, array_ty, false, MirPointerKind::SharedRef); + let (module_ptr, block) = build_kernel(&mut ctx, vec![], vec![]); + + let undef = mir::MirUndefOp::new(&mut ctx, array_ty); + undef.get_operation().insert_at_back(block, &ctx); + let value = undef.get_operation().deref(&ctx).get_result(0); + let ref_op = Operation::new( + &mut ctx, + mir::MirRefOp::get_concrete_op_info(), + vec![mir_ptr_ty.into()], + vec![value], + vec![], + 0, + ); + let mir_ref = mir::MirRefOp::new(ref_op); + mir_ref.set_mutable(&mut ctx, false); + mir_ref.set_pointer_kind_authority(&mut ctx, MirPointerKindAuthorityAttr::Reborrow); + ref_op.insert_at_back(block, &ctx); + append_mir_return(&mut ctx, block, vec![]); + + crate::lower_mir_to_llvm(&mut ctx, module_ptr).expect("lowering failed"); + + let body = kernel_blocks(&ctx, module_ptr); + let alloca = find_first::(&ctx, &body).expect("expected llvm.alloca"); + let store = find_first::(&ctx, &body).expect("expected llvm.store"); + let llvm_array_ty = alloca.result_pointee_type(&ctx); + let llvm_array_data = llvm_array_ty.deref(&ctx); + let llvm_array = llvm_array_data + .downcast_ref::() + .expect("over-aligned enum array must remain an LLVM array"); + + assert_eq!(llvm_array.size(), 3); + assert_eq!( + crate::convert::types::llvm_type_size_align(&ctx, llvm_array.elem_type()), + Some((abi_align, 16)), + "enum aligned {abi_align} keeps rustc's size at the capped natural alignment" + ); + assert_eq!( + crate::convert::types::llvm_type_size_align(&ctx, llvm_array_ty), + Some((abi_align * 3, 16)), + "enum array aligned {abi_align} keeps rustc's stride" + ); + assert_eq!( + llvm_export::ops::op_alignment(&ctx, alloca.get_operation()), + Some(abi_align as u32), + "alloca of an enum array aligned {abi_align} carries the Rust alignment" + ); + assert_eq!( + llvm_export::ops::op_alignment(&ctx, store.get_operation()), + Some(abi_align as u32), + "store of an enum array aligned {abi_align} carries the Rust alignment" + ); + } +} + /// Converted enum allocation size must equal rustc's `total_size` for /// every memory-faithful tag shape: that size is what GEP strides by. #[test] diff --git a/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs b/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs index b464ae3c75..3d6737580b 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs @@ -15,7 +15,7 @@ use pliron::builtin::types::{IntegerType, Signedness}; use pliron::context::Context; use pliron::r#type::TypeHandle; -use super::layout::{make_enum_filler_type, mir_type_contains_i1}; +use super::layout::{alignment_anchor_type, make_enum_filler_type, mir_type_contains_i1}; use super::pointer_storage::{ PointerOverlapRejection, analyze_pointer_overlap, llvm_type_contains_pointer, }; @@ -507,14 +507,14 @@ pub(crate) fn build_enum_slot_map( )); } if natural_align < required_align { - // Raise the storage's natural alignment with a zero-length anchor; - // it occupies no bytes, so the claim slots only shift by one. - // NVPTX scalar alignment tops out at 16 bytes. Cap the anchor before - // converting alignment to bit width; stronger Rust ABI alignment is - // carried explicitly by memory operations. - let anchor_align = required_align.min(16); - let anchor_int = IntegerType::get(ctx, (anchor_align * 8) as u32, Signedness::Signless); - let anchor: TypeHandle = llvm_types::ArrayType::get(ctx, anchor_int.into(), 0).into(); + // The byte claims alone can under-align the storage, e.g. when the + // only claim is an i8 niche carrier inside a 4-aligned enum. Raise + // the struct's alignment with a zero-length anchor, the same helper + // union storage uses; it occupies no bytes, so every slot index + // simply shifts by one. The anchor is capped at 16 bytes (see + // `alignment_anchor_type`); Rust alignment above that is stamped on + // the memory operations typed as the enum. + let anchor = alignment_anchor_type(ctx, required_align); llvm_fields.insert(0, anchor); for slot in &mut slot_of_claim { *slot += 1; diff --git a/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs b/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs index 2488ed3fe1..f985f45949 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs @@ -390,6 +390,46 @@ pub(crate) fn mir_type_abi_align(ctx: &Context, mir_ty: TypeHandle) -> Option TypeHandle { + let anchor_align = abi_align.clamp(1, MAX_NATURAL_ALIGN); + debug_assert!( + anchor_align.is_power_of_two(), + "alignment anchor needs a power-of-two ABI alignment, got {abi_align}" + ); + let bits = + u32::try_from(anchor_align * 8).expect("anchor is at most 16 bytes, so at most 128 bits"); + let anchor_int: TypeHandle = IntegerType::get(ctx, bits, Signedness::Signless).into(); + llvm_types::ArrayType::get(ctx, anchor_int, 0).into() +} + /// LLVM natural-layout `(size, align)` of an exported LLVM type, in bytes. /// /// Mirrors LLVM's default data layout for nvptx64 (scalars align to their @@ -402,8 +442,9 @@ pub(crate) fn llvm_type_size_align(ctx: &Context, ty: TypeHandle) -> Option<(u64 if let Some(int_ty) = ty_ref.downcast_ref::() { let size = (int_ty.width() as u64).div_ceil(8); - // i8 → 1, i16 → 2, i32 → 4, i64 → 8, i128 → 16. - return Some((size, size.next_power_of_two().min(16))); + // i8 → 1, i16 → 2, i32 → 4, i64 → 8, i128 → 16; wider integers stay + // at 16, see `MAX_NATURAL_ALIGN`. + return Some((size, size.next_power_of_two().min(MAX_NATURAL_ALIGN))); } if ty_ref.is::() { return Some((2, 2)); diff --git a/cuda-oxide/crates/mir-lower/src/convert/types/mod.rs b/cuda-oxide/crates/mir-lower/src/convert/types/mod.rs index fb1181fa60..94af573e5d 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/types/mod.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/types/mod.rs @@ -136,6 +136,10 @@ pub(crate) use layout::{ llvm_byte_faithful_twin, llvm_type_contains_i1, llvm_type_is_byte_faithful, llvm_type_size_align, mir_element_stride, mir_type_abi_align, natural_struct_layout, }; +// Outside this module the anchor cap is read only by test fixtures; a plain +// re-export would trip `unused_imports` in non-test builds. +#[cfg(test)] +pub(crate) use layout::MAX_NATURAL_ALIGN; pub(crate) use pointer_storage::{ llvm_packed_struct_contains_pointer_in_address_space, llvm_type_contains_pointer_in_address_space, diff --git a/cuda-oxide/crates/mir-lower/src/convert/types/union_storage.rs b/cuda-oxide/crates/mir-lower/src/convert/types/union_storage.rs index 355108e023..455769b517 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/types/union_storage.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/types/union_storage.rs @@ -7,11 +7,10 @@ use dialect_mir::types::MirUnionType; use llvm_export::types as llvm_types; -use pliron::builtin::types::{IntegerType, Signedness}; use pliron::context::Context; use pliron::r#type::TypeHandle; -use super::layout::make_padding_type; +use super::layout::{alignment_anchor_type, make_padding_type}; use super::pointer_storage::llvm_type_contains_pointer; use super::{convert_type, llvm_type_is_byte_faithful, llvm_type_size_align}; use crate::convert::target_stable_storage::{StorageRewriteOptions, target_stable_storage_type}; @@ -61,8 +60,9 @@ pub(crate) fn union_field_storage_type( /// alignment without consuming storage. Pointer-bearing fields are preferred /// so an ordinary union copy keeps LLVM pointer provenance. /// -/// NVPTX gives scalar integers natural alignments up to 16 bytes. Stronger -/// Rust alignment is carried explicitly on memory operations because LLVM +/// The anchor comes from [`alignment_anchor_type`], which caps it at +/// `MAX_NATURAL_ALIGN` (16 bytes) for the reasons given there. Stronger Rust +/// alignment is carried explicitly on memory operations because LLVM /// aggregate types cannot encode over-alignment; the storage type still keeps /// the union's exact size and therefore its array stride. pub(crate) fn build_union_storage_type( @@ -144,9 +144,7 @@ pub(crate) fn build_union_storage_type( fields.push((storage_field_ty, field_size, field_align, contains_pointer)); } - let storage_align = align.min(16); - let anchor_int = IntegerType::get(ctx, (storage_align * 8) as u32, Signedness::Signless); - let anchor: TypeHandle = llvm_types::ArrayType::get(ctx, anchor_int.into(), 0).into(); + let anchor = alignment_anchor_type(ctx, align); let mut storage_fields = vec![anchor]; if size > 0 { let representative = fields From cb24f4ae6ebeb41eca861fff4ecc36b4d204123d Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:50:35 +0530 Subject: [PATCH 3/5] test(examples): read a repr(align(32)) enum in repr_u32_enum_stride No tracked example read an over-aligned enum on a GPU, so the capped alignment anchor had no end-to-end coverage. Tag32 is a u8 tag padded to 32 bytes; the new kernel reads it through *const Tag32 over a byte buffer, so the 32-byte stride and the align 32 on the tag load both have to come from rustc's layout. The host checks tags at offsets 0, 32, 64 and 96. The PASS marker, manifest, lockfile and smoketest category are unchanged. Signed-off-by: nihalpasham --- .../examples/repr_u32_enum_stride/README.md | 7 +++ .../examples/repr_u32_enum_stride/src/main.rs | 56 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/README.md index 0d0caed544..3ef01ab567 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/README.md +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/README.md @@ -33,6 +33,7 @@ repr_c_enum [0, 1, 2, 3] PASS sparse_enum [0, 1000000, 1000000, 0] PASS neg_enum [-1, 0, -1, 0] PASS usize_enum [0, 1, 2, 3] PASS +align32_enum [0, 1, 1, 0] PASS RESULT: PASS - every enum shape strides and sign-extends correctly. ``` @@ -58,11 +59,17 @@ shape with one mechanism, which is why this example also tests: | `sparse_enum` | default repr, `B = 1_000_000` | u32 tag, stride 4 | | `neg_enum` | default repr, `N = -1` | SIGNED i8 tag, `sext` | | `usize_enum` | `#[repr(usize)]`, 4 unit variants | stride 8 | +| `align32_enum` | `#[repr(align(32))]`, 2 unit variants | u8 tag, stride 32, `align 32` on the load | `neg_enum` is the signedness half of the bug: the old model hardcoded an unsigned tag, so a memory-loaded `e as i32` zero-extended the `0xFF` byte to `255` instead of sign-extending it to `-1`. +`align32_enum` is the over-aligned shape: rustc pads the one-byte tag to 32 +bytes, and the LLVM storage type's zero-length alignment anchor is capped at +16 bytes, so both the stride and the `align 32` on the tag load must come +from rustc's layout rather than from the LLVM type. + Niched enums (for example `Option<&T>`) use a different rustc layout path: their otherwise-invalid payload value is recorded as the physical carrier, and cuda-oxide does not add a separate variant-count tag. diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs index 9964708571..d71f86e0c5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs @@ -41,6 +41,13 @@ * memory-loaded `e as i32` must sign-extend to -1 * usize_enum `#[repr(usize)]` tag is pointer-width (8 bytes) * + * One more shape checks alignment rather than the tag: + * + * align32_enum `#[repr(align(32))]`, a u8 tag plus 31 padding bytes. The + * LLVM storage type's alignment anchor is capped at 16, so + * the 32-byte stride and the `align 32` on the tag load + * must both come from rustc's layout + * * Build: * cargo oxide run repr_u32_enum_stride */ @@ -116,6 +123,20 @@ pub enum UTag { // SAFETY: trivial repr(usize) POD, safe to copy device↔host. unsafe impl cuda_core::DeviceCopy for UTag {} +/// Fieldless `#[repr(align(32))]` enum: a u8 tag followed by 31 padding +/// bytes, so `*const Tag32` must stride by 32 over a byte buffer. The LLVM +/// storage type carries a zero-length alignment anchor capped at 16 bytes; +/// the 32 rides on the memory operations. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(align(32))] +pub enum Tag32 { + A = 0, + B = 1, +} + +// SAFETY: fieldless POD enum, safe to copy device↔host. +unsafe impl cuda_core::DeviceCopy for Tag32 {} + const N: usize = 4; #[cuda_module] @@ -225,6 +246,23 @@ mod kernels { *slot = tag as u32; } } + + /// `#[repr(align(32))]` stride test: the enum is one tag byte plus 31 + /// padding bytes, so `*const Tag32` must stride by 32 over a byte buffer + /// and the tag load keeps its 32-byte alignment claim. + #[kernel] + pub fn read_via_align32_enum(input: &[u8], mut out: DisjointSlice) { + let idx = thread::index_1d(); + let i = idx.get(); + if i >= N { + return; + } + let base: *const Tag32 = input.as_ptr() as *const Tag32; + let tag = unsafe { *base.add(i) }; + if let Some(slot) = out.get_mut(idx) { + *slot = tag as u32; + } + } } /// Run one kernel over `input`, compare against `expected`, report PASS/FAIL. @@ -314,6 +352,24 @@ fn main() { unsafe { module.read_via_usize_enum(s, cfg, i, o) }.expect("launch") }); + // #[repr(align(32))]: one tag byte at the start of each 32-byte slot; + // stride 32 over a byte buffer. Device allocations are 256-byte aligned, + // so the 32-byte alignment the tag load claims holds. + let align32_tags: [u8; N] = [0, 1, 1, 0]; + let mut align32_in = vec![0u8; N * 32]; + for (slot, &tag) in align32_tags.iter().enumerate() { + align32_in[slot * 32] = tag; + } + let align32_expected: Vec = align32_tags.iter().map(|&tag| u32::from(tag)).collect(); + all_pass &= check( + "align32_enum", + &stream, + &align32_in, + &align32_expected, + // SAFETY: launch shape/resources match the kernel; buffers cover its accesses. + |s, cfg, i, o| unsafe { module.read_via_align32_enum(s, cfg, i, o) }.expect("launch"), + ); + println!(); if all_pass { println!("RESULT: PASS - every enum shape strides and sign-extends correctly."); From 25113a8bcf663a061648622806ca234f531c22db Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:50:35 +0530 Subject: [PATCH 4/5] fix(mir-lower): check the anchor's ABI alignment before clamping it The debug assertion ran after the clamp, so it could only ever see 1 to 16 and never caught a non-power-of-two ABI alignment above the cap. Signed-off-by: nihalpasham --- cuda-oxide/crates/mir-lower/src/convert/types/layout.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs b/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs index f985f45949..a01b237327 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/types/layout.rs @@ -419,11 +419,11 @@ pub(crate) const MAX_NATURAL_ALIGN: u64 = 16; /// alignment. When the claims already reach the cap and rustc asks for more, /// the anchor is a no-op field. pub(super) fn alignment_anchor_type(ctx: &mut Context, abi_align: u64) -> TypeHandle { - let anchor_align = abi_align.clamp(1, MAX_NATURAL_ALIGN); debug_assert!( - anchor_align.is_power_of_two(), + abi_align.is_power_of_two(), "alignment anchor needs a power-of-two ABI alignment, got {abi_align}" ); + let anchor_align = abi_align.clamp(1, MAX_NATURAL_ALIGN); let bits = u32::try_from(anchor_align * 8).expect("anchor is at most 16 bytes, so at most 128 bits"); let anchor_int: TypeHandle = IntegerType::get(ctx, bits, Signedness::Signless).into(); From e8e81bc4e55ffd397778ce48918a3acd428eeeb6 Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:50:35 +0530 Subject: [PATCH 5/5] docs(book): show the enum alignment anchor and its cap A one-page figure for the book and the pull request, in the style of the tile-libraries direct-linkage figure. Signed-off-by: nihalpasham --- .../compiler/images/enum-alignment-anchor.svg | 145 ++++++++++++++++++ .../compiler/lowering-pipeline.md | 15 ++ 2 files changed, 160 insertions(+) create mode 100644 cuda-oxide/cuda-oxide-book/compiler/images/enum-alignment-anchor.svg diff --git a/cuda-oxide/cuda-oxide-book/compiler/images/enum-alignment-anchor.svg b/cuda-oxide/cuda-oxide-book/compiler/images/enum-alignment-anchor.svg new file mode 100644 index 0000000000..947f4ff900 --- /dev/null +++ b/cuda-oxide/cuda-oxide-book/compiler/images/enum-alignment-anchor.svg @@ -0,0 +1,145 @@ + + + + + +Enum alignment anchors: the zero-length anchor is capped at 16 bytes ([0 x i128]) +mir-lower raises an enum storage type's natural alignment with a zero-length integer array. Its width was align * 8 with no bound; the cap is the one unions got in #610. +THE ENUM AND ITS ANCHOR (what mir-lower mints) +BEFORE (the width grows with the alignment, then breaks) +AFTER #1357 (one cap, stamped alignment, one helper) + +#[repr(align(536870912))] +pub enum HugeAlign { A, B } +rustc: size 2^29, align 2^29, u8 tag at offset 0 + +build_enum_slot_map mints the LLVM storage type + +{ [0 x iN], i8, [size - 1 x i8] } +[0 x iN] +0 bytes: only sets the struct's natural alignment + + + +i8 +[size - 1 x i8] +tag +filler to rustc's size + + + +size = align bytes; arrays stride by it; the anchor adds none +Here the tag and filler bytes give the struct natural +alignment 1. When rustc wants more, slot 0 gets an +anchor whose element width is align * 8. It adds no +bytes, so size and array stride stay rustc's; only +LLVM's idea of the type's alignment moves. + +Unions always carry one (union_storage.rs); an enum +gets one when its byte claims under-align it +(enum_layout.rs). Both are minted once per type. + +enum_layout.rs on main: width = align * 8, no bound +let anchor_int = IntegerType::get( + ctx, (required_align * 8) as u32, Signedness::Signless); +let anchor = ArrayType::get(ctx, anchor_int.into(), 0); +llvm_fields.insert(0, anchor); +The same three lines, with align.min(16), had capped unions since #610. +What the anchor became +align(N) +anchor on main +LLVM +anchor now + +8 +[0 x i64] +ok +[0 x i64] +16 +[0 x i128] +ok +[0 x i128] +32 +[0 x i256] +16 vs 32 +[0 x i128] +64 +[0 x i512] +16 vs 32 +[0 x i128] +128 +[0 x i1024] +16 vs 32 +[0 x i128] +2^20 +[0 x i8388608] +16 vs 32 +[0 x i128] +2^21 +[0 x i16777216] +rejected +[0 x i128] +2^28 +[0 x i2147483648] +rejected +[0 x i128] +2^29 +[0 x i0] +rejected +[0 x i128] + +ok + valid; the same anchor after the fix. Align 8 and 16 are the controls. +16 vs 32 + valid IR but no alignment gained: our data layout strings stop at + i128:128, so opt saw 16; llc's default NVPTX layout has i256:256: 32. +rejected + error: bitwidth for integer or byte type out of range (opt and llc), + build exit 101. From 2^21 up the width passes LLVM's 2^23-bit limit + (i8388608 is the last accepted); at 2^29, (2^29 * 8) as u32 == 0. + +the cap: anchor_align = min(align, 16) +main { [0 x i0], i8, [536870911 x i8] } +PR { [0 x i128], i8, [536870911 x i8] } +size 2^29, stride 2^29, tag at offset 0: unchanged. +Builds, runs and passes memcheck; main exits 101. + +the Rust alignment rides on the ops, not the anchor +alloca [3 x { [0 x i128], i8, [63 x i8] }], align 64 +store { [0 x i128], i8, [63 x i8] } .., align 64 +load i8, ptr %v13, align 536870912 +.local .align 64 .b8 __local_depot1[192] +.param .align 64 .b8 by_value_tag64_param_0[64] +shl.b64 %rd18, %rd3, 29 stride 2^29 +Stamped on every alloca, load, store, spill, kernel +parameter and return typed as the enum. PTX for align +32, 64 and 128 is instruction-identical to main. + +one helper for unions and enums +pub(crate) const MAX_NATURAL_ALIGN: u64 = 16; +pub(crate) fn alignment_anchor_type( + ctx: &mut Context, abi_align: u64) -> TypeHandle +Called from union_storage.rs and enum_layout.rs. +16 is where the three data layout strings we emit, +llvm_type_size_align and the exporter's +natural_alignment all stop (i128). + + + +What changes and what does not +Changes: only the anchor's element width for alignments of 32 and up (i256, i512, i1024, i8388608 and i0 all become i128). Unchanged: size, stride, and the alignment +stamped on the memory operations. Enums aligned 2^20, 2^21 and 2^29 go from exit 101 on main to building and running; align 32 to 128 keep the same PTX. A wider anchor never bought alignment. + diff --git a/cuda-oxide/cuda-oxide-book/compiler/lowering-pipeline.md b/cuda-oxide/cuda-oxide-book/compiler/lowering-pipeline.md index eb0cc7e4a4..e227a9e59c 100644 --- a/cuda-oxide/cuda-oxide-book/compiler/lowering-pipeline.md +++ b/cuda-oxide/cuda-oxide-book/compiler/lowering-pipeline.md @@ -229,6 +229,21 @@ It is not elegant, but it is exactly how C compilers have handled tagged unions for decades. LLVM's optimizer is quite good at cleaning up the redundant insertvalue/extractvalue chains. +An enum whose tag and payload bytes under-align its storage, a `repr(align(N))` +enum for example, also gets a zero-length integer array at slot 0 as an alignment +anchor. The anchor adds no bytes and is capped at 16 bytes; the Rust alignment +itself is stamped on every alloca, load and store of the enum. + +```{figure} images/enum-alignment-anchor.svg +:align: center +:width: 100% + +The enum alignment anchor. A zero-length integer array at slot 0 raises the storage type's natural +alignment without adding bytes. Its width used to be `align * 8` with no bound, which LLVM rejects +from align 2^21 up and which wraps to `i0` at 2^29. It is now capped at `[0 x i128]`, as union +storage has been since #610, while the Rust alignment rides on every memory operation. +``` + --- ## GPU Intrinsic Conversion