Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ use pliron::operation::Operation;
use pliron::r#type::TypeHandle;
use pliron::value::Value;

use crate::convert::types::MAX_NATURAL_ALIGN;

pub(super) fn insert_indices(ctx: &Context, inserts: &[llvm::InsertValueOp]) -> Vec<Vec<u32>> {
inserts.iter().map(|op| op.indices(ctx)).collect()
}
Expand Down Expand Up @@ -82,7 +84,7 @@ pub(super) fn unit_niche_enum(
.collect::<Vec<_>>();
let discriminants = (0..inhabited.len() as u64).collect();
let carrier_size = u64::from(carrier_width).div_ceil(8);
let carrier_align = carrier_size.next_power_of_two().min(16);
let carrier_align = carrier_size.next_power_of_two().min(MAX_NATURAL_ALIGN);
MirEnumType::get_with_encoding(
ctx,
"UnitNiche".into(),
Expand Down
142 changes: 142 additions & 0 deletions cuda-oxide/crates/mir-lower/src/convert/ops/memory/tests/enums.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,148 @@ fn unit_variants(n: usize) -> Vec<EnumVariant> {
(0..n).map(|i| EnumVariant::unit(format!("V{i}"))).collect()
}

/// The alignment anchor is `[0 x iN]` with `N = 8 * align` bits, so an
/// unbounded anchor failed in two ways: from `repr(align(2^21))` up the
/// width passed LLVM's 2^23-bit integer limit (`i8388608` is the last width
/// LLVM accepts, `i8388609` the first it rejects), and at `repr(align(2^29))`
/// the `u32` conversion wrapped to `i0`. `alignment_anchor_type` caps the
/// anchor at `MAX_NATURAL_ALIGN` (16 bytes), which is where cuda-oxide's data
/// layouts and layout model stop anyway; memory operations carry the stronger
/// Rust ABI alignment explicitly. The 8 and 16 rows are the controls, 2^20 is
/// the last width LLVM accepted, 2^21 the first it rejected, 2^29 the wrap.
#[test]
fn enum_conversion_handles_repr_align_bitwidth_boundary() {
use crate::convert::types::{llvm_type_size_align, mir_type_abi_align};

for (align, anchor_bits) in [
(8u64, 64),
(16, 128),
(32, 128),
(64, 128),
(1 << 20, 128),
(1 << 21, 128),
(1 << 29, 128),
] {
let mut ctx = make_ctx();
let ty = make_enum_ty(&mut ctx, "AlignedEnum", 8, unit_variants(2), align, align);
let conv = convert_type(&mut ctx, ty).unwrap();
let array_ty: TypeHandle = MirArrayType::get(&mut ctx, ty, 3).into();
let conv_array = convert_type(&mut ctx, array_ty).unwrap();
let natural_align = u64::from(anchor_bits / 8);

assert_eq!(
llvm_type_size_align(&ctx, conv),
Some((align, natural_align)),
"enum with {align}-byte alignment keeps rustc size"
);
assert_eq!(
llvm_type_size_align(&ctx, conv_array),
Some((align * 3, natural_align)),
"enum array with {align}-byte alignment keeps rustc stride"
);
assert_eq!(mir_type_abi_align(&ctx, ty), Some(align));
assert_eq!(mir_type_abi_align(&ctx, array_ty), Some(align));

let conv_ref = conv.deref(&ctx);
let struct_ty = conv_ref
.downcast_ref::<StructType>()
.expect("converted enum is a struct");
let anchor_ty = struct_ty
.fields()
.next()
.expect("expected alignment anchor");
let anchor_ref = anchor_ty.deref(&ctx);
let anchor = anchor_ref
.downcast_ref::<ArrayType>()
.expect("alignment anchor is an array");
assert_eq!(anchor.size(), 0, "alignment anchor must occupy no bytes");
let anchor_int = anchor.elem_type();
let anchor_int_ref = anchor_int.deref(&ctx);
assert_eq!(
anchor_int_ref
.downcast_ref::<IntegerType>()
.expect("alignment anchor element is an integer")
.width(),
anchor_bits,
"enum with {align}-byte alignment uses a bounded integer anchor"
);
}
}

/// Lowering a reference to an over-aligned enum array keeps the Rust ABI
/// alignment on the alloca and the store, while the storage type keeps
/// rustc's size and stride at the capped 16-byte natural alignment. Enum
/// sibling of `convert_ref_preserves_over_aligned_union_array_layout_and_alignment`
/// in `access.rs`; 2^29 is the alignment whose anchor used to wrap to `i0`.
#[test]
fn convert_ref_preserves_over_aligned_enum_array_layout_and_alignment() {
for abi_align in [32u64, 64, 1 << 29] {
let mut ctx = make_ctx();
let enum_ty = make_enum_ty(
&mut ctx,
"AlignedEnum",
8,
unit_variants(2),
abi_align,
abi_align,
);
let array_ty: TypeHandle = MirArrayType::get(&mut ctx, enum_ty, 3).into();
let mir_ptr_ty =
MirPtrType::get_generic_with_kind(&mut ctx, array_ty, false, MirPointerKind::SharedRef);
let (module_ptr, block) = build_kernel(&mut ctx, vec![], vec![]);

let undef = mir::MirUndefOp::new(&mut ctx, array_ty);
undef.get_operation().insert_at_back(block, &ctx);
let value = undef.get_operation().deref(&ctx).get_result(0);
let ref_op = Operation::new(
&mut ctx,
mir::MirRefOp::get_concrete_op_info(),
vec![mir_ptr_ty.into()],
vec![value],
vec![],
0,
);
let mir_ref = mir::MirRefOp::new(ref_op);
mir_ref.set_mutable(&mut ctx, false);
mir_ref.set_pointer_kind_authority(&mut ctx, MirPointerKindAuthorityAttr::Reborrow);
ref_op.insert_at_back(block, &ctx);
append_mir_return(&mut ctx, block, vec![]);

crate::lower_mir_to_llvm(&mut ctx, module_ptr).expect("lowering failed");

let body = kernel_blocks(&ctx, module_ptr);
let alloca = find_first::<llvm::AllocaOp>(&ctx, &body).expect("expected llvm.alloca");
let store = find_first::<llvm::StoreOp>(&ctx, &body).expect("expected llvm.store");
let llvm_array_ty = alloca.result_pointee_type(&ctx);
let llvm_array_data = llvm_array_ty.deref(&ctx);
let llvm_array = llvm_array_data
.downcast_ref::<ArrayType>()
.expect("over-aligned enum array must remain an LLVM array");

assert_eq!(llvm_array.size(), 3);
assert_eq!(
crate::convert::types::llvm_type_size_align(&ctx, llvm_array.elem_type()),
Some((abi_align, 16)),
"enum aligned {abi_align} keeps rustc's size at the capped natural alignment"
);
assert_eq!(
crate::convert::types::llvm_type_size_align(&ctx, llvm_array_ty),
Some((abi_align * 3, 16)),
"enum array aligned {abi_align} keeps rustc's stride"
);
assert_eq!(
llvm_export::ops::op_alignment(&ctx, alloca.get_operation()),
Some(abi_align as u32),
"alloca of an enum array aligned {abi_align} carries the Rust alignment"
);
assert_eq!(
llvm_export::ops::op_alignment(&ctx, store.get_operation()),
Some(abi_align as u32),
"store of an enum array aligned {abi_align} carries the Rust alignment"
);
}
}

/// Converted enum allocation size must equal rustc's `total_size` for
/// every memory-faithful tag shape: that size is what GEP strides by.
#[test]
Expand Down
13 changes: 7 additions & 6 deletions cuda-oxide/crates/mir-lower/src/convert/types/enum_layout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use pliron::builtin::types::{IntegerType, Signedness};
use pliron::context::Context;
use pliron::r#type::TypeHandle;

use super::layout::{make_enum_filler_type, mir_type_contains_i1};
use super::layout::{alignment_anchor_type, make_enum_filler_type, mir_type_contains_i1};
use super::pointer_storage::{
PointerOverlapRejection, analyze_pointer_overlap, llvm_type_contains_pointer,
};
Expand Down Expand Up @@ -509,11 +509,12 @@ pub(crate) fn build_enum_slot_map(
if natural_align < required_align {
// The byte claims alone can under-align the storage, e.g. when the
// only claim is an i8 niche carrier inside a 4-aligned enum. Raise
// the struct's alignment with a zero-length anchor field, the same
// mechanism union storage uses; it occupies no bytes, so every slot
// index simply shifts by one.
let anchor_int = IntegerType::get(ctx, (required_align * 8) as u32, Signedness::Signless);
let anchor: TypeHandle = llvm_types::ArrayType::get(ctx, anchor_int.into(), 0).into();
// the struct's alignment with a zero-length anchor, the same helper
// union storage uses; it occupies no bytes, so every slot index
// simply shifts by one. The anchor is capped at 16 bytes (see
// `alignment_anchor_type`); Rust alignment above that is stamped on
// the memory operations typed as the enum.
let anchor = alignment_anchor_type(ctx, required_align);
llvm_fields.insert(0, anchor);
for slot in &mut slot_of_claim {
*slot += 1;
Expand Down
45 changes: 43 additions & 2 deletions cuda-oxide/crates/mir-lower/src/convert/types/layout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,46 @@ pub(crate) fn mir_type_abi_align(ctx: &Context, mir_ty: TypeHandle) -> Option<u6
None
}

/// Widest natural alignment, in bytes, that cuda-oxide's LLVM layout model
/// gives any scalar: the alignment of `i128`.
///
/// Natural alignment is the alignment LLVM derives from a type alone, with
/// no `align` attribute on the operation. The three data layout strings the
/// exporter emits stop at `i128:128`, [`llvm_type_size_align`] gives every
/// wider integer this alignment, and the exporter's `natural_alignment`
/// declines above it. llc's default NVPTX data layout goes further
/// (`i256:256`), so a type whose natural alignment came from a wider integer
/// made `opt` (16) and llc (32) disagree and raised nothing cuda-oxide relies
/// on. Rust alignment above this bound rides on the memory operations.
pub(crate) const MAX_NATURAL_ALIGN: u64 = 16;

/// Zero-length field that raises an aggregate's natural alignment to
/// `abi_align` capped at [`MAX_NATURAL_ALIGN`]: `[0 x iN]`, where `N` is
/// eight times that byte count. It occupies no bytes, so inserting it at
/// slot 0 only shifts the other slots by one.
///
/// Union and enum storage insert it when the byte claims alone under-align
/// the storage, for example an `i8` tag inside a `repr(align(8))` enum. The
/// cap is what keeps the anchor valid: the width grows with the alignment,
/// so an uncapped anchor passed LLVM's 2^23-bit integer limit from
/// `repr(align(2097152))` up and wrapped to `i0` at `repr(align(536870912))`.
/// Above the cap the anchor would buy no alignment anyway (see
/// [`MAX_NATURAL_ALIGN`]); every alloca, load, store, spill, kernel parameter
/// and return typed as the enum or union itself is stamped with the Rust
/// alignment. When the claims already reach the cap and rustc asks for more,
/// the anchor is a no-op field.
pub(super) fn alignment_anchor_type(ctx: &mut Context, abi_align: u64) -> TypeHandle {
debug_assert!(
abi_align.is_power_of_two(),
"alignment anchor needs a power-of-two ABI alignment, got {abi_align}"
);
let anchor_align = abi_align.clamp(1, MAX_NATURAL_ALIGN);
let bits =
u32::try_from(anchor_align * 8).expect("anchor is at most 16 bytes, so at most 128 bits");
let anchor_int: TypeHandle = IntegerType::get(ctx, bits, Signedness::Signless).into();
llvm_types::ArrayType::get(ctx, anchor_int, 0).into()
}

/// LLVM natural-layout `(size, align)` of an exported LLVM type, in bytes.
///
/// Mirrors LLVM's default data layout for nvptx64 (scalars align to their
Expand All @@ -402,8 +442,9 @@ pub(crate) fn llvm_type_size_align(ctx: &Context, ty: TypeHandle) -> Option<(u64

if let Some(int_ty) = ty_ref.downcast_ref::<IntegerType>() {
let size = (int_ty.width() as u64).div_ceil(8);
// i8 → 1, i16 → 2, i32 → 4, i64 → 8, i128 → 16.
return Some((size, size.next_power_of_two().min(16)));
// i8 → 1, i16 → 2, i32 → 4, i64 → 8, i128 → 16; wider integers stay
// at 16, see `MAX_NATURAL_ALIGN`.
return Some((size, size.next_power_of_two().min(MAX_NATURAL_ALIGN)));
}
if ty_ref.is::<llvm_types::HalfType>() {
return Some((2, 2));
Expand Down
4 changes: 4 additions & 0 deletions cuda-oxide/crates/mir-lower/src/convert/types/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ pub(crate) use layout::{
llvm_byte_faithful_twin, llvm_type_contains_i1, llvm_type_is_byte_faithful,
llvm_type_size_align, mir_element_stride, mir_type_abi_align, natural_struct_layout,
};
// Outside this module the anchor cap is read only by test fixtures; a plain
// re-export would trip `unused_imports` in non-test builds.
#[cfg(test)]
pub(crate) use layout::MAX_NATURAL_ALIGN;
pub(crate) use pointer_storage::{
llvm_packed_struct_contains_pointer_in_address_space,
llvm_type_contains_pointer_in_address_space,
Expand Down
12 changes: 5 additions & 7 deletions cuda-oxide/crates/mir-lower/src/convert/types/union_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@

use dialect_mir::types::MirUnionType;
use llvm_export::types as llvm_types;
use pliron::builtin::types::{IntegerType, Signedness};
use pliron::context::Context;
use pliron::r#type::TypeHandle;

use super::layout::make_padding_type;
use super::layout::{alignment_anchor_type, make_padding_type};
use super::pointer_storage::llvm_type_contains_pointer;
use super::{convert_type, llvm_type_is_byte_faithful, llvm_type_size_align};
use crate::convert::target_stable_storage::{StorageRewriteOptions, target_stable_storage_type};
Expand Down Expand Up @@ -61,8 +60,9 @@ pub(crate) fn union_field_storage_type(
/// alignment without consuming storage. Pointer-bearing fields are preferred
/// so an ordinary union copy keeps LLVM pointer provenance.
///
/// NVPTX gives scalar integers natural alignments up to 16 bytes. Stronger
/// Rust alignment is carried explicitly on memory operations because LLVM
/// The anchor comes from [`alignment_anchor_type`], which caps it at
/// `MAX_NATURAL_ALIGN` (16 bytes) for the reasons given there. Stronger Rust
/// alignment is carried explicitly on memory operations because LLVM
/// aggregate types cannot encode over-alignment; the storage type still keeps
/// the union's exact size and therefore its array stride.
pub(crate) fn build_union_storage_type(
Expand Down Expand Up @@ -144,9 +144,7 @@ pub(crate) fn build_union_storage_type(
fields.push((storage_field_ty, field_size, field_align, contains_pointer));
}

let storage_align = align.min(16);
let anchor_int = IntegerType::get(ctx, (storage_align * 8) as u32, Signedness::Signless);
let anchor: TypeHandle = llvm_types::ArrayType::get(ctx, anchor_int.into(), 0).into();
let anchor = alignment_anchor_type(ctx, align);
let mut storage_fields = vec![anchor];
if size > 0 {
let representative = fields
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ repr_c_enum [0, 1, 2, 3] PASS
sparse_enum [0, 1000000, 1000000, 0] PASS
neg_enum [-1, 0, -1, 0] PASS
usize_enum [0, 1, 2, 3] PASS
align32_enum [0, 1, 1, 0] PASS
RESULT: PASS - every enum shape strides and sign-extends correctly.
```

Expand All @@ -58,11 +59,17 @@ shape with one mechanism, which is why this example also tests:
| `sparse_enum` | default repr, `B = 1_000_000` | u32 tag, stride 4 |
| `neg_enum` | default repr, `N = -1` | SIGNED i8 tag, `sext` |
| `usize_enum` | `#[repr(usize)]`, 4 unit variants | stride 8 |
| `align32_enum` | `#[repr(align(32))]`, 2 unit variants | u8 tag, stride 32, `align 32` on the load |

`neg_enum` is the signedness half of the bug: the old model hardcoded an
unsigned tag, so a memory-loaded `e as i32` zero-extended the `0xFF` byte to
`255` instead of sign-extending it to `-1`.

`align32_enum` is the over-aligned shape: rustc pads the one-byte tag to 32
bytes, and the LLVM storage type's zero-length alignment anchor is capped at
16 bytes, so both the stride and the `align 32` on the tag load must come
from rustc's layout rather than from the LLVM type.

Niched enums (for example `Option<&T>`) use a different rustc layout path:
their otherwise-invalid payload value is recorded as the physical carrier,
and cuda-oxide does not add a separate variant-count tag.
Loading
Loading