From e7b3b140d9aad21a4c7a3de20648e6492a3e5447 Mon Sep 17 00:00:00 2001
From: Yukang <moorekang@gmail.com>
Date: Tue, 22 Sep 2026 00:00:27 +0800
Subject: [PATCH 01/11] Add regression test for mutable closure argument
 suggestions

---
 tests/ui/suggestions/as-mut-closure.rs     |  33 ++++++
 tests/ui/suggestions/as-mut-closure.stderr | 111 +++++++++++++++++++++
 2 files changed, 144 insertions(+)
 create mode 100644 tests/ui/suggestions/as-mut-closure.rs
 create mode 100644 tests/ui/suggestions/as-mut-closure.stderr

diff --git a/tests/ui/suggestions/as-mut-closure.rs b/tests/ui/suggestions/as-mut-closure.rs
new file mode 100644
index 0000000000000..ab1c8bff5bf77
--- /dev/null
+++ b/tests/ui/suggestions/as-mut-closure.rs
@@ -0,0 +1,33 @@
+// Borrowing a container for a closure argument must preserve the expected mutability.
+
+#![allow(unused_mut)]
+
+fn append(value: &mut String) {
+    value.push('!');
+}
+
+fn increment(value: &mut i32) {
+    *value += 1;
+}
+
+fn main() {
+    let mut option = Some(String::new());
+    let _ = option.map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut option).and_then(|arg| Some(append(arg)));
+    //~^ ERROR mismatched types
+
+    let mut result: Result<_, ()> = Ok(String::new());
+    let _ = result.map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut result).and_then(|arg| Ok(append(arg)));
+    //~^ ERROR mismatched types
+
+    // A shared reference cannot supply `as_mut()`. Borrow the copied argument instead.
+    let shared = &Some(0);
+    let _ = shared.map(|mut arg| increment(arg));
+    //~^ ERROR mismatched types
+    let nested = &mut &Some(0);
+    let _ = nested.map(|mut arg| increment(arg));
+    //~^ ERROR mismatched types
+}
diff --git a/tests/ui/suggestions/as-mut-closure.stderr b/tests/ui/suggestions/as-mut-closure.stderr
new file mode 100644
index 0000000000000..c133641b6a5a7
--- /dev/null
+++ b/tests/ui/suggestions/as-mut-closure.stderr
@@ -0,0 +1,111 @@
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:15:37
+   |
+LL |     let _ = option.map(|arg| append(arg));
+   |                              ------ ^^^ expected `&mut String`, found `String`
+   |                              |
+   |                              arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:5:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_ref` instead
+   |
+LL |     let _ = option.as_ref().map(|arg| append(arg));
+   |                    +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:17:54
+   |
+LL |     let _ = (&mut option).and_then(|arg| Some(append(arg)));
+   |                                               ------ ^^^ expected `&mut String`, found `String`
+   |                                               |
+   |                                               arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:5:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_ref` instead
+   |
+LL |     let _ = (&mut option).as_ref().and_then(|arg| Some(append(arg)));
+   |                           +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:21:37
+   |
+LL |     let _ = result.map(|arg| append(arg));
+   |                              ------ ^^^ expected `&mut String`, found `String`
+   |                              |
+   |                              arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:5:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_ref` instead
+   |
+LL |     let _ = result.as_ref().map(|arg| append(arg));
+   |                    +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:23:52
+   |
+LL |     let _ = (&mut result).and_then(|arg| Ok(append(arg)));
+   |                                             ------ ^^^ expected `&mut String`, found `String`
+   |                                             |
+   |                                             arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:5:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_ref` instead
+   |
+LL |     let _ = (&mut result).as_ref().and_then(|arg| Ok(append(arg)));
+   |                           +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:28:44
+   |
+LL |     let _ = shared.map(|mut arg| increment(arg));
+   |                                  --------- ^^^ expected `&mut i32`, found integer
+   |                                  |
+   |                                  arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:9:4
+   |
+LL | fn increment(value: &mut i32) {
+   |    ^^^^^^^^^ ---------------
+help: consider using `as_ref` instead
+   |
+LL |     let _ = shared.as_ref().map(|mut arg| increment(arg));
+   |                    +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:31:44
+   |
+LL |     let _ = nested.map(|mut arg| increment(arg));
+   |                                  --------- ^^^ expected `&mut i32`, found integer
+   |                                  |
+   |                                  arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:9:4
+   |
+LL | fn increment(value: &mut i32) {
+   |    ^^^^^^^^^ ---------------
+help: consider using `as_ref` instead
+   |
+LL |     let _ = nested.as_ref().map(|mut arg| increment(arg));
+   |                    +++++++++
+
+error: aborting due to 6 previous errors
+
+For more information about this error, try `rustc --explain E0308`.

From a748eaedff6c55cab762add747a011271d93a0e8 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 22 Sep 2026 10:35:09 -0700
Subject: [PATCH 02/11] std: Update `wasip3` crate dependency

Keeping it up-to-date and resolving minor issues with it.
---
 library/Cargo.lock     | 12 ++++++------
 library/std/Cargo.toml |  4 ++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/library/Cargo.lock b/library/Cargo.lock
index 3ddddbd5985b7..98fa2c5117411 100644
--- a/library/Cargo.lock
+++ b/library/Cargo.lock
@@ -426,9 +426,9 @@ dependencies = [
 
 [[package]]
 name = "wasip2"
-version = "2.0.0+wasi-0.2.12"
+version = "2.0.1+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96744b5e833bfd2d84c6faa7569693ce9e04a47c0a4d5519e6442f18af1a0fed"
+checksum = "89aafd4b69fb41a64cfd5d7f214cde92ab422ec4d9bcd55fcc816d3f5f49bbf3"
 dependencies = [
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
@@ -437,9 +437,9 @@ dependencies = [
 
 [[package]]
 name = "wasip3"
-version = "0.8.0+wasi-0.3.0"
+version = "0.9.0+wasi-0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42a999271e77c083825863fc85404197523b3f3cebe9b402aece658ebea90065"
+checksum = "f1d5749fdf69bb9400562eba50f8f25f5cd800ef4b74300038b6f99ae7409674"
 dependencies = [
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
@@ -456,9 +456,9 @@ version = "0.61.100"
 
 [[package]]
 name = "wit-bindgen"
-version = "0.61.1"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e473fd0095479f9689ac7d2a52c427cc96bb2b973ace50238dfcc1ab1cd52d93"
+checksum = "53cb4b5556c3a791e86838ea287782bdafa704d55b0e68b5b81a3a16b9ea5f4b"
 dependencies = [
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml
index de5c1feadb298..1d35b42a53947 100644
--- a/library/std/Cargo.toml
+++ b/library/std/Cargo.toml
@@ -84,12 +84,12 @@ wasip1 = { version = "1.0.0", features = [
 ], default-features = false }
 
 [target.'cfg(all(target_os = "wasi", target_env = "p2"))'.dependencies]
-wasip2 = { version = '2.0.0', features = [
+wasip2 = { version = '2.0.1', features = [
     'rustc-dep-of-std',
 ], default-features = false }
 
 [target.'cfg(all(target_os = "wasi", target_env = "p3"))'.dependencies]
-wasip3 = { version = '0.8.0', features = [
+wasip3 = { version = '0.9.0', features = [
     'rustc-dep-of-std',
 ], default-features = false }
 

From 76c850e82b7ac6c81fefc4c1a82374f67ff02599 Mon Sep 17 00:00:00 2001
From: mejrs <59372212+mejrs@users.noreply.github.com>
Date: Tue, 22 Sep 2026 20:42:24 +0200
Subject: [PATCH 03/11] document `#[rustc_dyn_incompatible_trait]`

---
 compiler/rustc_attr_ir/src/attribute_docs.rs  | 60 +++++++++++++++++++
 compiler/rustc_attr_ir/src/data_structures.rs |  2 +-
 .../rustc_dyn_incompatible_trait.rs           |  8 +++
 .../rustc_dyn_incompatible_trait.stderr       | 18 ++++++
 .../rustc_dyn_incompatible_trait2.rs          | 52 ++++++++++++++++
 5 files changed, 139 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.rs
 create mode 100644 tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.stderr
 create mode 100644 tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs

diff --git a/compiler/rustc_attr_ir/src/attribute_docs.rs b/compiler/rustc_attr_ir/src/attribute_docs.rs
index 4d40cd6d52f16..a2b00c90b3028 100644
--- a/compiler/rustc_attr_ir/src/attribute_docs.rs
+++ b/compiler/rustc_attr_ir/src/attribute_docs.rs
@@ -223,3 +223,63 @@ const _: () = ();
 ///
 /// [`vtable_entries`]: ../rustc_middle/ty/struct.TyCtxt.html#method.vtable_entries
 const _: () = ();
+
+#[doc(attribute = "rustc_dyn_incompatible_trait", alias = "rustc_do_not_implement_via_object")]
+/// Opts a trait out of [dyn compatibility].
+///
+/// This is useful to reserve the ability to add dyn incompatible supertraits or methods to a trait
+/// in the future and to ensure the soundness of various constructs - see below for more about that.
+///
+/// For example [`Field`], [`FnPtr`], [`Tuple`], [`TransmuteFrom`], [`Sized`] and [`Unsize`] must
+/// be dyn incompatible because these traits describe properties and layouts of types that would
+/// be invalid for trait objects.
+///
+/// While making a trait dyn incompatible can also be done by including a (hidden and/or unstable)
+/// dyn incompatible method in the trait, using `#[rustc_dyn_incompatible_trait]` should be
+/// preferred because it is self-documenting and generates better error messages.
+///
+/// # Example
+///
+#[doc = include_example!("rustc_dyn_incompatible_trait")]
+///
+/// # Unsafe traits and dyn (in)compatibility
+///
+/// [Recall] that a trait object (`dyn Trait`) implements the base trait, its auto traits, and any supertraits of
+/// the base trait. This means that it's possible to run into subtle soundness problems when relying
+/// on the safety contract of a dyn compatible unsafe trait. See the following example:
+///
+// ignore-tidy-odd-backticks
+/// ```should_panic
+#[doc = include_str!("../../../tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs")]
+// ignore-tidy-odd-backticks
+/// ```
+///
+/// A solution for this is to make `UnsafeTrait` dyn incompatible, forcing `LocalTrait` to also be
+/// dyn incompatible so that a `dyn LocalTrait` cannot be formed. This is what we ended up doing for
+/// [#154619].
+///
+/// [`Allocator`] has had similar problems with  [`Clone`] ([#156920])
+/// but as we really wanted `dyn Allocator` to be a thing we ended up not making it dyn
+/// incompatible -- we ended up moving the safety contract to [`AllocatorClone`] instead.
+///
+/// See also [#156917] and [#160045] for more examples of this problem.
+///
+/// See [`AttributeKind::RustcDynIncompatibleTrait`] for the internal representation of this attribute.
+///
+/// [`Allocator`]: core::alloc::Allocator
+/// [`AllocatorClone`]: core::alloc::AllocatorClone
+/// [`Clone`]: core::clone::Clone
+/// [`Field`]: core::field::Field
+// FIXME: use core::ops::FnPtr once trickled down to beta
+/// [`FnPtr`]: https://doc.rust-lang.org/nightly/core/ops/trait.FnPtr.html
+/// [`Tuple`]: core::marker::Tuple
+/// [`TransmuteFrom`]: core::mem::TransmuteFrom
+/// [`Sized`]: core::marker::Sized
+/// [`Unsize`]: core::marker::Unsize
+/// [dyn compatibility]: https://doc.rust-lang.org/nightly/reference/items/traits.html#dyn-compatibility
+/// [recall]: https://doc.rust-lang.org/nightly/reference/types/trait-object.html#r-type.trait-object.impls
+/// [#154619]: https://github.com/rust-lang/rust/issues/154619 "`deref_patterns` is unsound due to `dyn` of subtrait of `DerefPure`"
+/// [#156917]: https://github.com/rust-lang/rust/issues/156917 "`dyn Allocator` together with `Allocator + PartialEq` safety requirements leads to unsoundness"
+/// [#156920]:https://github.com/rust-lang/rust/issues/156920 "`dyn Allocator` together with `Allocator + Clone` requirements is unsound, leading to UB with `Arc`"
+/// [#160045]:https://github.com/rust-lang/rust/issues/160045 "`iter::Rev`'s `TrustedLen` impl is unsound with trait objects"
+const _: () = ();
diff --git a/compiler/rustc_attr_ir/src/data_structures.rs b/compiler/rustc_attr_ir/src/data_structures.rs
index d722d515582dc..712ed41ae7759 100644
--- a/compiler/rustc_attr_ir/src/data_structures.rs
+++ b/compiler/rustc_attr_ir/src/data_structures.rs
@@ -1267,7 +1267,7 @@ pub enum AttributeKind {
     /// Represents the [`rustc_dump_vtable`](./attribute.rustc_dump_vtable.html) attribute.
     RustcDumpVtable(Span),
 
-    /// Represents `#[rustc_dyn_incompatible_trait]`.
+    /// Represents the [`rustc_dyn_incompatible_trait`](./attribute.rustc_dyn_incompatible_trait.html) attribute.
     RustcDynIncompatibleTrait(Span),
 
     /// Represents `#[rustc_effective_visibility]`.
diff --git a/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.rs b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.rs
new file mode 100644
index 0000000000000..912cdef5b60bb
--- /dev/null
+++ b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.rs
@@ -0,0 +1,8 @@
+//@ dont-require-annotations: ERROR
+//@ compile-flags: --crate-type lib -Z ui-testing=no
+#![feature(rustc_attrs)]
+
+#[rustc_dyn_incompatible_trait]
+pub trait DynIncompatible {}
+
+pub fn f(_x: &dyn DynIncompatible) {}
diff --git a/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.stderr b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.stderr
new file mode 100644
index 0000000000000..c871061a57507
--- /dev/null
+++ b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.stderr
@@ -0,0 +1,18 @@
+error[E0038]: the trait `DynIncompatible` is not dyn compatible
+ --> $DIR/rustc_dyn_incompatible_trait.rs:8:15
+  |
+8 | pub fn f(_x: &dyn DynIncompatible) {}
+  |               ^^^^^^^^^^^^^^^^^^^ `DynIncompatible` is not dyn compatible
+  |
+note: for a trait to be dyn compatible it needs to allow building a vtable
+      for more information, visit <https://doc.rust-lang.org/reference/items/traits.html#dyn-compatibility>
+ --> $DIR/rustc_dyn_incompatible_trait.rs:5:1
+  |
+5 | #[rustc_dyn_incompatible_trait]
+  | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...because it opted out of dyn-compatibility
+6 | pub trait DynIncompatible {}
+  |           --------------- this trait is not dyn compatible...
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0038`.
diff --git a/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs
new file mode 100644
index 0000000000000..fd8b3ec4e2761
--- /dev/null
+++ b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs
@@ -0,0 +1,52 @@
+//@run-fail
+mod foreign_crate {
+    /// # Safety requirements
+    ///
+    /// If this type also implements `SafeTrait`,
+    /// then that implementation must always return `true`.
+    pub unsafe trait UnsafeTrait {}
+    unsafe impl<T: UnsafeTrait + ?Sized> UnsafeTrait for &T {}
+
+    pub trait SafeTrait {
+        fn returns_true(&self) -> bool;
+    }
+    impl<T: SafeTrait + ?Sized> SafeTrait for &T {
+        fn returns_true(&self) -> bool {
+            (*self).returns_true()
+        }
+    }
+
+    impl SafeTrait for u8 {
+        fn returns_true(&self) -> bool {
+            true
+        }
+    }
+    /// Safety: impl returns `true`.
+    unsafe impl UnsafeTrait for u8 {}
+
+    pub fn function(x: impl UnsafeTrait + SafeTrait) {
+        // Can't panic, after all, `x: UnsafeTrait`
+        // guarantees `returns_true` actually returns `true`
+        assert!(x.returns_true());
+    }
+}
+
+use foreign_crate::{SafeTrait, UnsafeTrait, function};
+
+pub trait LocalTrait: UnsafeTrait {}
+impl<T: UnsafeTrait> LocalTrait for T {}
+
+// We can do this because `dyn LocalTrait` is a local type.
+// But `LocalTrait: UnsafeTrait`, so `dyn LocalTrait: UnsafeTrait` holds,
+// and we don't have to `unsafe impl` it.
+impl SafeTrait for dyn LocalTrait {
+    fn returns_true(&self) -> bool {
+        false
+    }
+}
+
+fn main() {
+    let x = 42_u8;
+    let y: &dyn LocalTrait = &x;
+    function(y); // panics
+}

From 9b9cb70640a38e006a36b86e91975c6659802475 Mon Sep 17 00:00:00 2001
From: Manuel Drehwald <git@manuel.drehwald.info>
Date: Tue, 22 Sep 2026 16:44:04 -0400
Subject: [PATCH 04/11] Stop embedding device code, now that the
 clang-linker-wrapper isn't consuming it anymore

---
 compiler/rustc_codegen_llvm/src/back/write.rs   | 16 +---------------
 compiler/rustc_codegen_llvm/src/diagnostics.rs  |  4 ----
 .../rustc_codegen_llvm/src/llvm/offload_ffi.rs  | 16 ----------------
 .../llvm-wrapper/offload/OffloadWrapper.cpp     | 17 -----------------
 4 files changed, 1 insertion(+), 52 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index 92989ba2dcf46..fa11a6abe6ed2 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -824,9 +824,7 @@ pub(crate) unsafe fn llvm_optimize(
 
     // This assumes that we previously compiled our kernels for a gpu target, which created a
     // `device.bin` artifact. The user is supposed to provide us with a path to this artifact, we
-    // don't need any other artifacts from the previous run. We will embed this artifact into our
-    // LLVM-IR host module, to create a `host.o` ObjectFile, which we will write to disk.
-    // The last, not yet automated steps uses the `clang-linker-wrapper` to process `host.o`.
+    // don't need any other artifacts from the previous run.
     if !cgcx.target_is_like_gpu && is_final_stage {
         if let Some(device_path) = config
             .offload
@@ -850,18 +848,6 @@ pub(crate) unsafe fn llvm_optimize(
             let out_obj = host_dir.join("host.o");
             let device_bin_c = path_to_c_string(device_pathbuf.as_path());
 
-            // 2) Finalize host: lib.bc + device.bin -> host.o (host TM)
-            // We create a full clone of our LLVM host module, since we will embed the device IR
-            // into it, and this might break caching or incremental compilation otherwise.
-            let ok = unsafe {
-                llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_embed_buffer_in_module(
-                    module.module_llvm.llmod(),
-                    device_bin_c.as_c_str(),
-                )
-            };
-            if !ok {
-                dcx.emit_err(crate::diagnostics::OffloadEmbedFailed);
-            }
             write_output_file(
                 dcx,
                 module.module_llvm.tm.raw(),
diff --git a/compiler/rustc_codegen_llvm/src/diagnostics.rs b/compiler/rustc_codegen_llvm/src/diagnostics.rs
index 78e502f13af75..d589c157cb9f0 100644
--- a/compiler/rustc_codegen_llvm/src/diagnostics.rs
+++ b/compiler/rustc_codegen_llvm/src/diagnostics.rs
@@ -101,10 +101,6 @@ pub(crate) struct OffloadNonexistingPath;
 #[diag("call to BundleImages failed, `device.bin` was not created")]
 pub(crate) struct OffloadBundleImagesFailed;
 
-#[derive(Diagnostic)]
-#[diag("call to EmbedBufferInModule failed, `host.o` was not created")]
-pub(crate) struct OffloadEmbedFailed;
-
 #[derive(Diagnostic)]
 #[diag("call to WrapImages failed, device image was not wrapped into the host module")]
 pub(crate) struct OffloadWrapImagesFailed;
diff --git a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs
index fecb9e40eab88..da3da6bd7abd4 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs
@@ -5,7 +5,6 @@ use std::sync::OnceLock;
 use super::ffi::{Module, TargetMachine, Value};
 
 type LLVMRustBundleImagesFn = unsafe extern "C" fn(&Module, &TargetMachine, *const c_char) -> bool;
-type LLVMRustOffloadEmbedBufferInModuleFn = unsafe extern "C" fn(&Module, *const c_char) -> bool;
 type LLVMRustOffloadMapperFn = unsafe extern "C" fn(&Value, &Value, *const &Value);
 type LLVMRustOffloadWrapImagesFn =
     unsafe extern "C" fn(&Module, *const c_char, *const c_char) -> bool;
@@ -18,7 +17,6 @@ use crate::llvm;
 
 pub(crate) struct RustOffloadWrapper {
     LLVMRustBundleImages: LLVMRustBundleImagesFn,
-    LLVMRustOffloadEmbedBufferInModule: LLVMRustOffloadEmbedBufferInModuleFn,
     LLVMRustOffloadMapper: LLVMRustOffloadMapperFn,
     LLVMRustOffloadWrapImages: LLVMRustOffloadWrapImagesFn,
     lld_path: Option<PathBuf>,
@@ -65,14 +63,6 @@ impl RustOffloadWrapper {
         unsafe { (self.LLVMRustBundleImages)(m, tm, c.as_ptr()) }
     }
 
-    pub(crate) unsafe fn llvm_rust_offload_embed_buffer_in_module(
-        &self,
-        m: &Module,
-        i: &CStr,
-    ) -> bool {
-        unsafe { (self.LLVMRustOffloadEmbedBufferInModule)(m, i.as_ptr()) }
-    }
-
     pub(crate) unsafe fn llvm_rust_offload_wrapper(&self, v1: &Value, v2: &Value, vs: &[&Value]) {
         unsafe { (self.LLVMRustOffloadMapper)(v1, v2, vs.as_ptr()) }
     }
@@ -96,11 +86,6 @@ impl RustOffloadWrapper {
 
         let llvm_rust_bundle_images =
             *unsafe { lib.get::<LLVMRustBundleImagesFn>(b"LLVMRustBundleImages\0")? };
-        let llvm_rust_offload_embed_buffer_in_module = *unsafe {
-            lib.get::<LLVMRustOffloadEmbedBufferInModuleFn>(
-                b"LLVMRustOffloadEmbedBufferInModule\0",
-            )?
-        };
         let llvm_rust_offload_wrapper =
             *unsafe { lib.get::<LLVMRustOffloadMapperFn>(b"LLVMRustOffloadMapper\0")? };
         let llvm_rust_offload_wrap_images =
@@ -108,7 +93,6 @@ impl RustOffloadWrapper {
 
         Ok(Self {
             LLVMRustBundleImages: llvm_rust_bundle_images,
-            LLVMRustOffloadEmbedBufferInModule: llvm_rust_offload_embed_buffer_in_module,
             LLVMRustOffloadMapper: llvm_rust_offload_wrapper,
             LLVMRustOffloadWrapImages: llvm_rust_offload_wrap_images,
             lld_path,
diff --git a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp
index bed54da0a7045..ca4d181179ffd 100644
--- a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp
@@ -28,7 +28,6 @@
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/Cloning.h"
-#include "llvm/Transforms/Utils/ModuleUtils.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 
 #include <optional>
@@ -88,22 +87,6 @@ extern "C" bool LLVMRustBundleImages(LLVMModuleRef M, TargetMachine &TM,
   return true;
 }
 
-extern "C" bool LLVMRustOffloadEmbedBufferInModule(LLVMModuleRef HostM,
-                                                   const char *HostOutPath) {
-  auto MBOrErr = MemoryBuffer::getFile(HostOutPath);
-  if (!MBOrErr) {
-    auto E = MBOrErr.getError();
-    auto _B = errorCodeToError(E);
-    return false;
-  }
-  MemoryBufferRef Buf = (*MBOrErr)->getMemBufferRef();
-  Module *M = unwrap(HostM);
-  StringRef SectionName = ".llvm.offloading";
-  Align Alignment = Align(8);
-  llvm::embedBufferInModule(*M, Buf, SectionName, Alignment);
-  return true;
-}
-
 // Clone OldFn into NewFn, remapping its arguments to RebuiltArgs.
 // Each arg of OldFn is replaced with the corresponding value in RebuiltArgs.
 // For scalars, RebuiltArgs contains the value cast and/or truncated to the

From 4f20be313591a2bd1404bd85676ce38895330c77 Mon Sep 17 00:00:00 2001
From: Manuel Drehwald <git@manuel.drehwald.info>
Date: Tue, 22 Sep 2026 16:50:51 -0400
Subject: [PATCH 05/11] also drop now unused host.o

---
 compiler/rustc_codegen_llvm/src/back/write.rs | 19 -------------------
 .../offload/host-std-device-nostd/rmake.rs    |  2 --
 2 files changed, 21 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index fa11a6abe6ed2..233e4a6b06612 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -843,26 +843,7 @@ pub(crate) unsafe fn llvm_optimize(
             } else if !device_pathbuf.exists() {
                 dcx.emit_err(crate::diagnostics::OffloadNonexistingPath);
             }
-            let host_path = cgcx.output_filenames.path(OutputType::Object);
-            let host_dir = host_path.parent().unwrap();
-            let out_obj = host_dir.join("host.o");
             let device_bin_c = path_to_c_string(device_pathbuf.as_path());
-
-            write_output_file(
-                dcx,
-                module.module_llvm.tm.raw(),
-                config.no_builtins,
-                module.module_llvm.llmod(),
-                &out_obj,
-                None,
-                llvm::FileType::ObjectFile,
-                prof,
-                true,
-            );
-            // We ignore cgcx.save_temps here and unconditionally always keep our `device.bin` artifact.
-            // Otherwise, recompiling the host code would fail since we deleted that device artifact
-            // in the previous host compilation, which would be confusing at best.
-
             let ok = unsafe {
                 llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_wrap_images(
                     module.module_llvm.llmod(),
diff --git a/tests/run-make/offload/host-std-device-nostd/rmake.rs b/tests/run-make/offload/host-std-device-nostd/rmake.rs
index 35f408e8b1cd0..6769993f9cf98 100644
--- a/tests/run-make/offload/host-std-device-nostd/rmake.rs
+++ b/tests/run-make/offload/host-std-device-nostd/rmake.rs
@@ -35,6 +35,4 @@ fn main() {
         .arg("-Clto=fat")
         .emit("obj")
         .run();
-
-    assert!(cwd().join("host.o").exists());
 }

From 87e95d69da85bc2c54432240cff00fb56c439eb2 Mon Sep 17 00:00:00 2001
From: Yukang <moorekang@gmail.com>
Date: Tue, 22 Sep 2026 00:10:54 +0800
Subject: [PATCH 06/11] Enhance mutable closure suggestions with as_mut()
 support

---
 .../src/fn_ctxt/suggestions.rs                | 33 ++++++++----
 tests/ui/suggestions/as-mut-closure.fixed     | 34 ++++++++++++
 tests/ui/suggestions/as-mut-closure.rs        |  1 +
 tests/ui/suggestions/as-mut-closure.stderr    | 52 +++++++++----------
 4 files changed, 84 insertions(+), 36 deletions(-)
 create mode 100644 tests/ui/suggestions/as-mut-closure.fixed

diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index 8895faef0890f..49f7b2e551b78 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -2855,7 +2855,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         true
     }
 
-    /// Identify some cases where `as_ref()` would be appropriate and suggest it.
+    /// Identify some cases where `as_ref()` or `as_mut()` would be appropriate and suggest it.
     ///
     /// Given the following code:
     /// ```compile_fail,E0308
@@ -2871,7 +2871,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// ```ignore (illustrative)
     /// opt.map(|param| { takes_ref(param) });
     /// ```
-    fn can_use_as_ref(&self, expr: &hir::Expr<'_>) -> Option<(Vec<(Span, String)>, &'static str)> {
+    fn can_use_as_ref_or_mut(
+        &self,
+        expr: &hir::Expr<'_>,
+        mutability: hir::Mutability,
+    ) -> Option<(Vec<(Span, String)>, &'static str)> {
         let hir::ExprKind::Path(hir::QPath::Resolved(_, path)) = expr.kind else {
             return None;
         };
@@ -2908,9 +2912,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return None;
         };
 
-        let self_ty = self.typeck_results.borrow().expr_ty_opt(receiver)?;
+        let mut self_ty = self.typeck_results.borrow().expr_ty_opt(receiver)?;
+        while let ty::Ref(_, inner, ref_mutability) = self_ty.kind() {
+            // `as_mut()` cannot borrow through a shared reference,
+            // also we cannot suggest `as_ref()` either when the reference is shared
+            if mutability.is_mut() && ref_mutability.is_not() {
+                return None;
+            }
+            self_ty = *inner;
+        }
         let name = method_path.ident.name;
-        let is_as_ref_able = match self_ty.peel_refs().kind() {
+        let can_borrow = match self_ty.kind() {
             ty::Adt(def, _) => {
                 (self.tcx.is_diagnostic_item(sym::Option, def.did())
                     || self.tcx.is_diagnostic_item(sym::Result, def.did()))
@@ -2918,11 +2930,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
             _ => false,
         };
-        if is_as_ref_able {
-            Some((
-                vec![(method_path.ident.span.shrink_to_lo(), "as_ref().".to_string())],
-                "consider using `as_ref` instead",
-            ))
+        if can_borrow {
+            let (suggestion, message) = match mutability {
+                hir::Mutability::Not => ("as_ref().", "consider using `as_ref` instead"),
+                hir::Mutability::Mut => ("as_mut().", "consider using `as_mut` instead"),
+            };
+            Some((vec![(method_path.ident.span.shrink_to_lo(), suggestion.to_string())], message))
         } else {
             None
         }
@@ -3113,7 +3126,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         return Some((suggs, help, app, mutref));
                     }
 
-                    if let Some((sugg, msg)) = self.can_use_as_ref(expr) {
+                    if let Some((sugg, msg)) = self.can_use_as_ref_or_mut(expr, mutability) {
                         return Some((
                             sugg,
                             msg.to_string(),
diff --git a/tests/ui/suggestions/as-mut-closure.fixed b/tests/ui/suggestions/as-mut-closure.fixed
new file mode 100644
index 0000000000000..76a9cc5ab7c88
--- /dev/null
+++ b/tests/ui/suggestions/as-mut-closure.fixed
@@ -0,0 +1,34 @@
+// Borrowing a container for a closure argument must preserve the expected mutability.
+//@ run-rustfix
+
+#![allow(unused_mut)]
+
+fn append(value: &mut String) {
+    value.push('!');
+}
+
+fn increment(value: &mut i32) {
+    *value += 1;
+}
+
+fn main() {
+    let mut option = Some(String::new());
+    let _ = option.as_mut().map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut option).as_mut().and_then(|arg| Some(append(arg)));
+    //~^ ERROR mismatched types
+
+    let mut result: Result<_, ()> = Ok(String::new());
+    let _ = result.as_mut().map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut result).as_mut().and_then(|arg| Ok(append(arg)));
+    //~^ ERROR mismatched types
+
+    // A shared reference cannot supply `as_mut()`. Borrow the copied argument instead.
+    let shared = &Some(0);
+    let _ = shared.map(|mut arg| increment(&mut arg));
+    //~^ ERROR mismatched types
+    let nested = &mut &Some(0);
+    let _ = nested.map(|mut arg| increment(&mut arg));
+    //~^ ERROR mismatched types
+}
diff --git a/tests/ui/suggestions/as-mut-closure.rs b/tests/ui/suggestions/as-mut-closure.rs
index ab1c8bff5bf77..4b583cbc4f2dc 100644
--- a/tests/ui/suggestions/as-mut-closure.rs
+++ b/tests/ui/suggestions/as-mut-closure.rs
@@ -1,4 +1,5 @@
 // Borrowing a container for a closure argument must preserve the expected mutability.
+//@ run-rustfix
 
 #![allow(unused_mut)]
 
diff --git a/tests/ui/suggestions/as-mut-closure.stderr b/tests/ui/suggestions/as-mut-closure.stderr
index c133641b6a5a7..8a998be539458 100644
--- a/tests/ui/suggestions/as-mut-closure.stderr
+++ b/tests/ui/suggestions/as-mut-closure.stderr
@@ -1,5 +1,5 @@
 error[E0308]: mismatched types
-  --> $DIR/as-mut-closure.rs:15:37
+  --> $DIR/as-mut-closure.rs:16:37
    |
 LL |     let _ = option.map(|arg| append(arg));
    |                              ------ ^^^ expected `&mut String`, found `String`
@@ -7,17 +7,17 @@ LL |     let _ = option.map(|arg| append(arg));
    |                              arguments to this function are incorrect
    |
 note: function defined here
-  --> $DIR/as-mut-closure.rs:5:4
+  --> $DIR/as-mut-closure.rs:6:4
    |
 LL | fn append(value: &mut String) {
    |    ^^^^^^ ------------------
-help: consider using `as_ref` instead
+help: consider using `as_mut` instead
    |
-LL |     let _ = option.as_ref().map(|arg| append(arg));
+LL |     let _ = option.as_mut().map(|arg| append(arg));
    |                    +++++++++
 
 error[E0308]: mismatched types
-  --> $DIR/as-mut-closure.rs:17:54
+  --> $DIR/as-mut-closure.rs:18:54
    |
 LL |     let _ = (&mut option).and_then(|arg| Some(append(arg)));
    |                                               ------ ^^^ expected `&mut String`, found `String`
@@ -25,17 +25,17 @@ LL |     let _ = (&mut option).and_then(|arg| Some(append(arg)));
    |                                               arguments to this function are incorrect
    |
 note: function defined here
-  --> $DIR/as-mut-closure.rs:5:4
+  --> $DIR/as-mut-closure.rs:6:4
    |
 LL | fn append(value: &mut String) {
    |    ^^^^^^ ------------------
-help: consider using `as_ref` instead
+help: consider using `as_mut` instead
    |
-LL |     let _ = (&mut option).as_ref().and_then(|arg| Some(append(arg)));
+LL |     let _ = (&mut option).as_mut().and_then(|arg| Some(append(arg)));
    |                           +++++++++
 
 error[E0308]: mismatched types
-  --> $DIR/as-mut-closure.rs:21:37
+  --> $DIR/as-mut-closure.rs:22:37
    |
 LL |     let _ = result.map(|arg| append(arg));
    |                              ------ ^^^ expected `&mut String`, found `String`
@@ -43,17 +43,17 @@ LL |     let _ = result.map(|arg| append(arg));
    |                              arguments to this function are incorrect
    |
 note: function defined here
-  --> $DIR/as-mut-closure.rs:5:4
+  --> $DIR/as-mut-closure.rs:6:4
    |
 LL | fn append(value: &mut String) {
    |    ^^^^^^ ------------------
-help: consider using `as_ref` instead
+help: consider using `as_mut` instead
    |
-LL |     let _ = result.as_ref().map(|arg| append(arg));
+LL |     let _ = result.as_mut().map(|arg| append(arg));
    |                    +++++++++
 
 error[E0308]: mismatched types
-  --> $DIR/as-mut-closure.rs:23:52
+  --> $DIR/as-mut-closure.rs:24:52
    |
 LL |     let _ = (&mut result).and_then(|arg| Ok(append(arg)));
    |                                             ------ ^^^ expected `&mut String`, found `String`
@@ -61,17 +61,17 @@ LL |     let _ = (&mut result).and_then(|arg| Ok(append(arg)));
    |                                             arguments to this function are incorrect
    |
 note: function defined here
-  --> $DIR/as-mut-closure.rs:5:4
+  --> $DIR/as-mut-closure.rs:6:4
    |
 LL | fn append(value: &mut String) {
    |    ^^^^^^ ------------------
-help: consider using `as_ref` instead
+help: consider using `as_mut` instead
    |
-LL |     let _ = (&mut result).as_ref().and_then(|arg| Ok(append(arg)));
+LL |     let _ = (&mut result).as_mut().and_then(|arg| Ok(append(arg)));
    |                           +++++++++
 
 error[E0308]: mismatched types
-  --> $DIR/as-mut-closure.rs:28:44
+  --> $DIR/as-mut-closure.rs:29:44
    |
 LL |     let _ = shared.map(|mut arg| increment(arg));
    |                                  --------- ^^^ expected `&mut i32`, found integer
@@ -79,17 +79,17 @@ LL |     let _ = shared.map(|mut arg| increment(arg));
    |                                  arguments to this function are incorrect
    |
 note: function defined here
-  --> $DIR/as-mut-closure.rs:9:4
+  --> $DIR/as-mut-closure.rs:10:4
    |
 LL | fn increment(value: &mut i32) {
    |    ^^^^^^^^^ ---------------
-help: consider using `as_ref` instead
+help: consider mutably borrowing here
    |
-LL |     let _ = shared.as_ref().map(|mut arg| increment(arg));
-   |                    +++++++++
+LL |     let _ = shared.map(|mut arg| increment(&mut arg));
+   |                                            ++++
 
 error[E0308]: mismatched types
-  --> $DIR/as-mut-closure.rs:31:44
+  --> $DIR/as-mut-closure.rs:32:44
    |
 LL |     let _ = nested.map(|mut arg| increment(arg));
    |                                  --------- ^^^ expected `&mut i32`, found integer
@@ -97,14 +97,14 @@ LL |     let _ = nested.map(|mut arg| increment(arg));
    |                                  arguments to this function are incorrect
    |
 note: function defined here
-  --> $DIR/as-mut-closure.rs:9:4
+  --> $DIR/as-mut-closure.rs:10:4
    |
 LL | fn increment(value: &mut i32) {
    |    ^^^^^^^^^ ---------------
-help: consider using `as_ref` instead
+help: consider mutably borrowing here
    |
-LL |     let _ = nested.as_ref().map(|mut arg| increment(arg));
-   |                    +++++++++
+LL |     let _ = nested.map(|mut arg| increment(&mut arg));
+   |                                            ++++
 
 error: aborting due to 6 previous errors
 

From e5e0b1057f9579198f4c536dcd2c2f7ad933cf75 Mon Sep 17 00:00:00 2001
From: Zac Harrold <zac@harrold.com.au>
Date: Wed, 23 Sep 2026 13:44:47 +1000
Subject: [PATCH 07/11] Fix typo in
 std::sys::process::unix::unsupported::wait_status documentation

---
 library/std/src/sys/process/unix/unsupported/wait_status.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/std/src/sys/process/unix/unsupported/wait_status.rs b/library/std/src/sys/process/unix/unsupported/wait_status.rs
index ac54824875813..b77e7ae835656 100644
--- a/library/std/src/sys/process/unix/unsupported/wait_status.rs
+++ b/library/std/src/sys/process/unix/unsupported/wait_status.rs
@@ -1,4 +1,4 @@
-//! Emulated wait status for non-Unix #[cfg(unix) platforms
+//! Emulated wait status for non-Unix `#[cfg(unix)]` platforms
 //!
 //! Separate module to facilitate testing against a real Unix implementation.
 

From 16e81e97d9747e4da7d68e38234cb9a0e62197e5 Mon Sep 17 00:00:00 2001
From: Amar Shah <github@amari.ca>
Date: Tue, 22 Sep 2026 22:49:09 -0700
Subject: [PATCH 08/11] Fix bignum build on 16-bit targets

---
 library/core/src/num/imp/bignum.rs                  |  6 +++---
 library/core/src/num/imp/flt2dec/strategy/dragon.rs | 10 +++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/library/core/src/num/imp/bignum.rs b/library/core/src/num/imp/bignum.rs
index 95046e96fc8f2..5c2a19f39dfce 100644
--- a/library/core/src/num/imp/bignum.rs
+++ b/library/core/src/num/imp/bignum.rs
@@ -386,7 +386,7 @@ macro_rules! define_bignum {
 /// the digit type for `Big32x40`
 pub type Digit32 = u32;
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 define_bignum!(Big32x40: type=Digit32, n=40);
 
 /// The digit type for `Big64x20`.
@@ -395,12 +395,12 @@ pub type Digit64 = u64;
 #[cfg(target_pointer_width = "64")]
 define_bignum!(Big64x20: type=Digit64, n=20);
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 pub type Big = Big32x40;
 #[cfg(target_pointer_width = "64")]
 pub type Big = Big64x20;
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 pub type Digit = Digit32;
 #[cfg(target_pointer_width = "64")]
 pub type Digit = Digit64;
diff --git a/library/core/src/num/imp/flt2dec/strategy/dragon.rs b/library/core/src/num/imp/flt2dec/strategy/dragon.rs
index 4ae0d4b4ab0ba..b63b93034d57f 100644
--- a/library/core/src/num/imp/flt2dec/strategy/dragon.rs
+++ b/library/core/src/num/imp/flt2dec/strategy/dragon.rs
@@ -16,22 +16,22 @@ static POW10: [Digit; 10] =
     [1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000];
 // precalculated arrays of `Digit`s for 5^(2^n).
 // FIXME(#162879): these tables have u64 and u32 versions, future versions may be generated by macro.
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO16: [Digit; 2] = [0x86f26fc1, 0x23];
 #[cfg(target_pointer_width = "64")]
 static POW5TO16: [Digit; 1] = [0x2386f26fc1];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO32: [Digit; 3] = [0x85acef81, 0x2d6d415b, 0x4ee];
 #[cfg(target_pointer_width = "64")]
 static POW5TO32: [Digit; 2] = [0x2d6d415b85acef81, 0x4ee];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO64: [Digit; 5] = [0xbf6a1f01, 0x6e38ed64, 0xdaa797ed, 0xe93ff9f4, 0x184f03];
 #[cfg(target_pointer_width = "64")]
 static POW5TO64: [Digit; 3] = [0x6e38ed64bf6a1f01, 0xe93ff9f4daa797ed, 0x184f03];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO128: [Digit; 10] = [
     0x2e953e01, 0x3df9909, 0xf1538fd, 0x2374e42f, 0xd3cff5ec, 0xc404dc08, 0xbccdb0da, 0xa6337f19,
     0xe91f2603, 0x24e,
@@ -40,7 +40,7 @@ static POW5TO128: [Digit; 10] = [
 static POW5TO128: [Digit; 5] =
     [0x3df99092e953e01, 0x2374e42f0f1538fd, 0xc404dc08d3cff5ec, 0xa6337f19bccdb0da, 0x24ee91f2603];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO256: [Digit; 19] = [
     0x982e7c01, 0xbed3875b, 0xd8d99f72, 0x12152f87, 0x6bde50c6, 0xcf4a6e70, 0xd595d80f, 0x26b2716e,
     0xadc666b0, 0x1d153624, 0x3c42d35a, 0x63ff540e, 0xcc5573c0, 0x65f9ef17, 0x55bc28f2, 0x80dcc7f7,

From 039b8bcadc7bcce087a278668f2bf7a56d7cc60c Mon Sep 17 00:00:00 2001
From: Flakebi <flakebi@t-online.de>
Date: Tue, 1 Sep 2026 09:40:37 +0200
Subject: [PATCH 09/11] Add address_space and byref to abi PassMode::Indirect
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both will be used by the amdgpu target to implement the `gpu-kernel`
ABI.

`address_space` specifies the address space of an indirect argument.

`AmdgpuKernelArg` translates to LLVM’s byref, which is similar to
on_stack/byval, however, there is no extra copy made, the pointer may
not point to the stack but can point to some other address space, and
the passed argument should not be modified.

byval and byref are mutually exclusive, so change on_stack to an enum
with the new states, Pointer (none), OnStack and AmdgpuKernelArg.
---
 compiler/rustc_abi/src/layout/ty.rs           |   4 +-
 .../src/abi/pass_mode.rs                      |  24 ++--
 .../src/abi/returning.rs                      |  17 +--
 compiler/rustc_codegen_gcc/src/abi.rs         |  33 ++++-
 compiler/rustc_codegen_llvm/src/abi.rs        | 111 +++++++++++++---
 compiler/rustc_codegen_llvm/src/llvm/ffi.rs   |   1 +
 compiler/rustc_codegen_llvm/src/llvm/mod.rs   |   4 +
 compiler/rustc_codegen_ssa/src/mir/block.rs   |  48 ++++---
 compiler/rustc_codegen_ssa/src/mir/mod.rs     |  16 ++-
 .../rustc_llvm/llvm-wrapper/RustWrapper.cpp   |   5 +
 .../src/deduce_param_attrs.rs                 |   2 +-
 compiler/rustc_public/src/abi.rs              |  18 ++-
 .../src/unstable/convert/stable/abi.rs        |  35 +++--
 compiler/rustc_target/src/callconv/mod.rs     | 125 ++++++++++++++----
 compiler/rustc_target/src/callconv/x86.rs     |   5 +-
 compiler/rustc_target/src/callconv/xtensa.rs  |   6 +-
 compiler/rustc_ty_utils/src/abi.rs            |  14 +-
 tests/assembly-llvm/tail-call-indirect.rs     |   6 +-
 tests/ui-fulldeps/rustc_public/check_abi.rs   |   8 +-
 .../rustc_public/check_abi_cast.rs            |   4 +-
 tests/ui/abi/c-zst.powerpc-linux.stderr       |   3 +-
 tests/ui/abi/c-zst.s390x-linux.stderr         |   3 +-
 tests/ui/abi/c-zst.sparc-linux.stderr         |   3 +-
 tests/ui/abi/c-zst.sparc-none.stderr          |   3 +-
 tests/ui/abi/c-zst.sparc64-linux.stderr       |   3 +-
 .../ui/abi/c-zst.x86_64-pc-windows-gnu.stderr |   3 +-
 tests/ui/abi/debug.generic.stderr             |   6 +-
 tests/ui/abi/debug.loongarch64.stderr         |   6 +-
 tests/ui/abi/debug.riscv64.stderr             |   6 +-
 tests/ui/abi/pass-indirectly-attr.rs          |   2 +-
 tests/ui/abi/pass-indirectly-attr.stderr      |   3 +-
 .../pass-by-value-abi.aarch64.stderr          |   3 +-
 tests/ui/c-variadic/pass-by-value-abi.rs      |   8 +-
 .../pass-by-value-abi.x86_64.stderr           |   9 +-
 tests/ui/explicit-tail-calls/indirect.rs      |  10 +-
 35 files changed, 406 insertions(+), 151 deletions(-)

diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs
index e51255dc5963f..b8928aecf0cc5 100644
--- a/compiler/rustc_abi/src/layout/ty.rs
+++ b/compiler/rustc_abi/src/layout/ty.rs
@@ -233,8 +233,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
     }
 
     /// If this method returns `true`, then this type should always have a `PassMode` of
-    /// `Indirect { on_stack: false, .. }` when being used as the argument type of a function with a
-    /// non-Rustic ABI (this is true for structs annotated with the
+    /// `Indirect { mode: IndirectMode::Pointer, .. }` when being used as the argument type of a
+    /// function with a non-Rustic ABI (this is true for structs annotated with the
     /// `#[rustc_pass_indirectly_in_non_rustic_abis]` attribute).
     ///
     /// This is used to replicate some of the behaviour of C array-to-pointer decay; however unlike
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 1c552ca1a9c32..48ffc43c5cfa1 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -3,7 +3,7 @@
 use cranelift_codegen::ir::ArgumentPurpose;
 use rustc_abi::{Reg, RegKind};
 use rustc_target::callconv::{
-    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode,
+    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, IndirectMode, PassMode,
 };
 use smallvec::{SmallVec, smallvec};
 
@@ -126,8 +126,12 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 assert_eq!(pad_i32_count, 0, "padding support not yet implemented");
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
             }
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
-                if on_stack {
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
+                assert!(
+                    mode != IndirectMode::AmdgpuKernelArg,
+                    "unsupported amdgpu kernel argument"
+                );
+                if mode == IndirectMode::OnStack {
                     // Abi requires aligning struct size to pointer size
                     let size = self.layout.size.align_to(tcx.data_layout.pointer_align().abi);
                     let size = u32::try_from(size.bytes()).unwrap();
@@ -139,8 +143,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                     smallvec![apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)]
                 }
             }
-            PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), address_space: _, mode } => {
+                assert!(mode == IndirectMode::Pointer);
                 smallvec![
                     apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs),
                     apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs),
@@ -184,8 +188,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 None,
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(),
             ),
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
+                assert!(mode == IndirectMode::Pointer);
                 (
                     Some(apply_attrs_to_abi_param(
                         AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn),
@@ -194,7 +198,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                     vec![],
                 )
             }
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 unreachable!("unsized return value")
             }
         }
@@ -324,7 +328,7 @@ pub(super) fn cvalue_for_param<'tcx>(
         PassMode::Cast { ref cast, .. } => {
             from_casted_value(fx, &block_params, arg_abi.layout, cast)
         }
-        PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => {
             assert_eq!(block_params.len(), 1, "{:?}", block_params);
             if let Some(pointee_align) = attrs.pointee_align
                 && pointee_align < arg_abi.layout.align.abi
@@ -342,7 +346,7 @@ pub(super) fn cvalue_for_param<'tcx>(
                 CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
             CValue::by_ref_unsized(Pointer::new(block_params[0]), block_params[1], arg_abi.layout)
         }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index 36087f96dd776..7f4ee9435b506 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -17,12 +17,12 @@ pub(super) fn codegen_return_param<'tcx>(
             let is_ssa = ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.ret.layout.ty);
             (super::make_local_place(fx, RETURN_PLACE, fx.fn_abi.ret.layout, is_ssa), smallvec![])
         }
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             let ret_param = block_params_iter.next().unwrap();
             assert_eq!(fx.bcx.func.dfg.value_type(ret_param), fx.pointer_type);
             (CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.ret.layout), smallvec![ret_param])
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
     };
@@ -50,7 +50,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
 ) {
     let (ret_temp_place, return_ptr) = match ret_arg_abi.mode {
         PassMode::Ignore => (None, None),
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             if let Some(ret_ptr) = ret_place.try_to_ptr() {
                 // This is an optimization to prevent unnecessary copies of the return value when
                 // the return place is already a memory place as opposed to a register.
@@ -61,7 +61,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
                 (Some(place), Some(place.to_ptr().get_addr(fx)))
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None),
@@ -86,14 +86,14 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
                 super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast);
             ret_place.write_cvalue(fx, result);
         }
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             if let Some(ret_temp_place) = ret_temp_place {
                 // If ret_temp_place is None, it is not necessary to copy the return value.
                 let ret_temp_value = ret_temp_place.to_cvalue(fx);
                 ret_place.write_cvalue(fx, ret_temp_value);
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
     }
@@ -102,10 +102,11 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
 /// Codegen a return instruction with the right return value(s) if any.
 pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
     match fx.fn_abi.ret.mode {
-        PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Ignore
+        | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             fx.bcx.ins().return_(&[]);
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
         PassMode::Direct(_) => {
diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs
index 63eaf52ce9f01..5b88cebb4f174 100644
--- a/compiler/rustc_codegen_gcc/src/abi.rs
+++ b/compiler/rustc_codegen_gcc/src/abi.rs
@@ -11,7 +11,7 @@ use rustc_middle::ty::layout::LayoutOf;
 #[cfg(feature = "master")]
 use rustc_session::{Session, config};
 use rustc_span::bug;
-use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode};
 #[cfg(feature = "master")]
 use rustc_target::spec::Arch;
 
@@ -189,7 +189,12 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     let ty = cast.gcc_type(cx);
                     apply_attrs(ty, &cast.attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs: _,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let x86_interrupt_first_arg = {
                         #[cfg(feature = "master")]
                         {
@@ -216,14 +221,32 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         arg.layout.gcc_type(cx)
                     }
                 }
+                PassMode::Indirect {
+                    attrs: _,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    unimplemented!("unsupported amdgpu kernel argument")
+                }
                 PassMode::Direct(attrs) => {
                     apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     apply_attrs(cx.type_ptr_to(arg.layout.gcc_type(cx)), &attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                    assert!(!on_stack);
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode,
+                } => {
+                    assert!(mode == IndirectMode::Pointer);
                     // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
                     // Any two ABI-compatible unsized types have the same metadata type and
                     // moreover the same metadata value leads to the same dynamic size and
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index dce8db6841b7f..e16f34483a0c5 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -15,7 +15,7 @@ use rustc_middle::ty::layout::LayoutOf;
 use rustc_session::{Session, config};
 use rustc_span::bug;
 use rustc_target::callconv::{
-    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode,
+    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, IndirectMode, PassMode,
 };
 use rustc_target::spec::{Arch, SanitizerSet};
 use smallvec::SmallVec;
@@ -243,12 +243,12 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         match &self.mode {
             PassMode::Ignore => {}
             // Sized indirect arguments
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => {
                 let align = attrs.pointee_align.unwrap_or(self.layout.align.abi);
                 OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst);
             }
             // Unsized indirect arguments cannot be stored
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 bug!("unsized `ArgAbi` cannot be stored");
             }
             PassMode::Cast { cast, pad_i32_count: _ } => {
@@ -304,11 +304,11 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             PassMode::Pair(..) => {
                 OperandValue::Pair(next(), next()).store(bx, dst);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 bug!("unsized `ArgAbi` cannot be stored");
             }
             PassMode::Direct(_)
-            | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ }
+            | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ }
             | PassMode::Cast { .. } => {
                 let next_arg = next();
                 self.store(bx, next_arg, dst);
@@ -369,8 +369,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Ignore => cx.type_void(),
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
             PassMode::Cast { cast, pad_i32_count: _ } => cast.llvm_type(cx),
-            PassMode::Indirect { .. } => {
-                llargument_tys.push(cx.type_ptr());
+            PassMode::Indirect { address_space, .. } => {
+                let ty = if let Some(address_space) = address_space {
+                    cx.type_ptr_ext(*address_space)
+                } else {
+                    cx.type_ptr()
+                };
+                llargument_tys.push(ty);
                 cx.type_void()
             }
         };
@@ -395,7 +400,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                     // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
                     // Any two ABI-compatible unsized types have the same metadata type and
                     // moreover the same metadata value leads to the same dynamic size and
@@ -406,7 +411,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(),
+                PassMode::Indirect { attrs: _, meta_attrs: None, address_space, mode: _ } => {
+                    if let Some(address_space) = address_space {
+                        cx.type_ptr_ext(*address_space)
+                    } else {
+                        cx.type_ptr()
+                    }
+                }
                 PassMode::Cast { cast, pad_i32_count } => {
                     // Add padding.
                     llargument_tys.extend(std::iter::repeat_n(
@@ -496,8 +507,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     apply_range_attr(llvm::AttributePlace::ReturnValue, scalar);
                 }
             }
-            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => {
+                assert!(*mode == IndirectMode::Pointer);
                 let i = apply(attrs);
                 let sret = llvm::CreateStructRetAttr(
                     cx.llcx,
@@ -523,7 +534,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         for arg in self.args.iter() {
             match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let i = apply(attrs);
                     let byval = llvm::CreateByValAttr(
                         cx.llcx,
@@ -531,13 +547,31 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     );
                     attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
                 }
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    let i = apply(attrs);
+                    let byref = llvm::CreateByRefAttr(
+                        cx.llcx,
+                        cx.type_array(cx.type_i8(), arg.layout.size.bytes()),
+                    );
+                    attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byref]);
+                }
                 PassMode::Direct(attrs) => {
                     let i = apply(attrs);
                     if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
                         apply_range_attr(llvm::AttributePlace::Argument(i), scalar);
                     }
                 }
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     let i = apply(attrs);
                     if cx.sess().opts.optimize != config::OptLevel::No {
                         attributes::apply_to_llfn(
@@ -547,8 +581,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         );
                     }
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                    assert!(!on_stack);
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode,
+                } => {
+                    assert!(*mode == IndirectMode::Pointer);
                     apply(attrs);
                     apply(meta_attrs);
                 }
@@ -626,8 +665,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Direct(attrs) => {
                 attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite);
             }
-            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => {
+                assert!(*mode == IndirectMode::Pointer);
                 let i = apply(bx.cx, attrs);
                 let sret = llvm::CreateStructRetAttr(
                     bx.cx.llcx,
@@ -647,7 +686,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         for arg in self.args.iter() {
             match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let i = apply(bx.cx, attrs);
                     let byval = llvm::CreateByValAttr(
                         bx.cx.llcx,
@@ -659,11 +703,38 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         &[byval],
                     );
                 }
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    let i = apply(bx.cx, attrs);
+                    let byref = llvm::CreateByRefAttr(
+                        bx.cx.llcx,
+                        bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()),
+                    );
+                    attributes::apply_to_callsite(
+                        callsite,
+                        llvm::AttributePlace::Argument(i),
+                        &[byref],
+                    );
+                }
                 PassMode::Direct(attrs)
-                | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                | PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     apply(bx.cx, attrs);
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack: _ } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode: _,
+                } => {
                     apply(bx.cx, attrs);
                     apply(bx.cx, meta_attrs);
                 }
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 52218bfa336b8..07381745afe84 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -2016,6 +2016,7 @@ unsafe extern "C" {
     pub(crate) fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute;
     pub(crate) fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute;
     pub(crate) fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
+    pub(crate) fn LLVMRustCreateByRefAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute;
diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
index 5452f4abc5c33..89e4d60656d34 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
@@ -122,6 +122,10 @@ pub(crate) fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll At
     unsafe { LLVMRustCreateByValAttr(llcx, ty) }
 }
 
+pub(crate) fn CreateByRefAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute {
+    unsafe { LLVMRustCreateByRefAttr(llcx, ty) }
+}
+
 pub(crate) fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute {
     unsafe { LLVMRustCreateStructRetAttr(llcx, ty) }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 6b0def4ffa182..f99009a0f4243 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -18,7 +18,7 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
 use rustc_session::config::OptLevel;
 use rustc_span::{Span, Spanned, bug, span_bug};
-use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode};
 use tracing::{debug, info};
 
 use super::operand::OperandRef;
@@ -1257,7 +1257,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             (args, None)
         };
 
-        // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+        // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
         //
         // Normally an indirect argument that is allocated in the caller's stack frame
         // would be passed as a pointer into the callee's stack frame.
@@ -1282,10 +1282,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let mut tail_call_temporaries = vec![];
         if kind == CallKind::Tail {
             tail_call_temporaries = vec![None; first_args.len()];
-            // Copy the arguments that use `PassMode::Indirect { on_stack: false , ..}`
+            // Copy the arguments that use `PassMode::Indirect { mode: IndirectMode::Pointer , ..}`
             // to temporary stack allocations. See the comment above.
             for (i, arg) in first_args.iter().enumerate() {
-                if !matches!(fn_abi.args[i].mode, PassMode::Indirect { on_stack: false, .. }) {
+                if !matches!(
+                    fn_abi.args[i].mode,
+                    PassMode::Indirect { mode: IndirectMode::Pointer, .. }
+                ) {
                     continue;
                 }
 
@@ -1353,10 +1356,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            let by_move = if let PassMode::Indirect { on_stack: false, .. } = fn_abi.args[i].mode
+            let by_move = if let PassMode::Indirect { mode: IndirectMode::Pointer, .. } =
+                fn_abi.args[i].mode
                 && kind == CallKind::Tail
             {
-                // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+                // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
                 //
                 // Normally an indirect argument that is allocated in the caller's stack frame
                 // would be passed as a pointer into the callee's stack frame.
@@ -1977,14 +1981,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
                 _ => bug!("codegen_argument: {:?} invalid for pair argument", op),
             },
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => match op.val {
-                Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => {
-                    llargs.push(a);
-                    llargs.push(b);
-                    return;
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
+                match op.val {
+                    Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => {
+                        llargs.push(a);
+                        llargs.push(b);
+                        return;
+                    }
+                    _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
                 }
-                _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
-            },
+            }
             _ => {}
         }
 
@@ -2014,7 +2020,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 PassMode::Ignore | PassMode::Pair(..) => unreachable!("handled above"),
             },
             Ref(op_place_val) => match arg.mode {
-                PassMode::Indirect { attrs, on_stack, .. } => {
+                PassMode::Indirect { attrs, mode, .. } => {
+                    if mode == IndirectMode::AmdgpuKernelArg {
+                        bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}");
+                    }
                     // For `foo(packed.large_field)`, and types with <4 byte alignment on x86,
                     // alignment requirements may be higher than the type's alignment, so copy
                     // to a higher-aligned alloca.
@@ -2023,7 +2032,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         None => arg.layout.align.abi,
                     };
                     // Copy to an alloca when the argument is neither by-val nor by-move.
-                    if op_place_val.align < required_align || (!on_stack && !by_move) {
+                    if op_place_val.align < required_align
+                        || (mode == IndirectMode::Pointer && !by_move)
+                    {
                         let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
                         bx.lifetime_start(scratch.llval, arg.layout.size);
                         op.store_with_annotation(bx, scratch.with_type(arg.layout));
@@ -2036,8 +2047,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 _ => (op_place_val.llval, op_place_val.align, true),
             },
             ZeroSized => match arg.mode {
-                PassMode::Indirect { on_stack, .. } => {
-                    if on_stack {
+                PassMode::Indirect { mode, .. } => {
+                    if mode == IndirectMode::AmdgpuKernelArg {
+                        bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}");
+                    }
+                    if mode == IndirectMode::OnStack {
                         // It doesn't seem like any target can have `byval` ZSTs, so this assert
                         // is here to replace a would-be untested codepath.
                         bug!("ZST {op:?} passed on stack with abi {arg:?}");
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index b5cecf4b5c434..aefa8356536dc 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -8,7 +8,7 @@ use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_span::{ErrorGuaranteed, bug, span_bug};
-use rustc_target::callconv::{FnAbi, PassMode};
+use rustc_target::callconv::{FnAbi, IndirectMode, PassMode};
 use tracing::{debug, instrument};
 
 use crate::base;
@@ -561,15 +561,21 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             match arg.mode {
                 // Sized indirect arguments
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+                PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
                     // Don't copy an indirect argument to an alloca, the caller already put it
                     // in a temporary alloca and gave it up.
+                    // AmdgpuKernelArg/byref arguments must not be modified, so always create a
+                    // local alloca for them.
+                    // If the argument is underaligned, then we need to copy it to a higher-aligned
+                    // alloca.
                     // FIXME: lifetimes
+                    let mut needs_alloca = mode == IndirectMode::AmdgpuKernelArg;
                     if let Some(pointee_align) = attrs.pointee_align
                         && pointee_align < arg.layout.align.abi
                     {
-                        // ...unless the argument is underaligned, then we need to copy it to
-                        // a higher-aligned alloca.
+                        needs_alloca = true;
+                    }
+                    if needs_alloca {
                         let tmp = PlaceRef::alloca(bx, arg.layout);
                         bx.store_fn_arg(arg, &mut llarg_idx, tmp);
                         LocalRef::Place(tmp)
@@ -580,7 +586,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                     }
                 }
                 // Unsized indirect arguments
-                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                     // As the storage for the indirect argument lives during
                     // the whole function call, we just copy the wide pointer.
                     let llarg = bx.get_param(llarg_idx);
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index 161b5bdb952d3..bc8fa60b66a52 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -480,6 +480,11 @@ extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C,
   return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty)));
 }
 
+extern "C" LLVMAttributeRef LLVMRustCreateByRefAttr(LLVMContextRef C,
+                                                    LLVMTypeRef Ty) {
+  return wrap(Attribute::getWithByRefType(*unwrap(C), unwrap(Ty)));
+}
+
 extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C,
                                                         LLVMTypeRef Ty) {
   return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty)));
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index 5bba125aefc58..8814670ca4300 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -135,7 +135,7 @@ impl<'tcx> Visitor<'tcx> for DeduceParamAttrs {
             }
 
             // Like a call, but more conservative because the backend may introduce writes to an
-            // argument if the argument is passed as `PassMode::Indirect { on_stack: false, ... }`.
+            // argument if the argument is passed as `PassMode::Indirect { mode: IndirectMode::Pointer, ... }`.
             TerminatorKind::TailCall { .. } => {
                 for usage in self.usage.iter_mut() {
                     *usage |= UsageSummary::MUTATE;
diff --git a/compiler/rustc_public/src/abi.rs b/compiler/rustc_public/src/abi.rs
index 67d609c780c42..72387fce19797 100644
--- a/compiler/rustc_public/src/abi.rs
+++ b/compiler/rustc_public/src/abi.rs
@@ -41,6 +41,19 @@ pub struct ArgAbi {
     pub mode: PassMode,
 }
 
+/// Different modes in which indirect arguments can be passed.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize)]
+pub enum IndirectMode {
+    /// Passed as a normal pointer, nothing special.
+    Pointer,
+    /// The value is placed at a fixed stack offset rather than passed as a regular pointer
+    /// argument.
+    OnStack,
+    /// Similar to `OnStack` except that the pointer does not necessarily point to the stack, no
+    /// extra copy is made, and the passed argument should not be modified.
+    AmdgpuKernelArg,
+}
+
 /// How a function argument should be passed in to the target function.
 ///
 /// The pass mode is determined by the platform's calling convention and the
@@ -74,14 +87,13 @@ pub enum PassMode {
     /// Pass the argument indirectly via a pointer.
     ///
     /// The caller places the value in memory and passes a pointer to it.
-    /// When `on_stack` is true, the value is placed at a fixed stack offset
-    /// rather than passed as a regular pointer argument.
     Indirect {
         attrs: ArgAttributes,
         /// Attributes for the metadata pointer (vtable or length) of unsized arguments.
         /// Only present for unsized types (e.g., `dyn Trait`, `[T]`).
         meta_attrs: Option<ArgAttributes>,
-        on_stack: bool,
+        address_space: Option<AddressSpace>,
+        mode: IndirectMode,
     },
 }
 
diff --git a/compiler/rustc_public/src/unstable/convert/stable/abi.rs b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
index 3c268a6dd23a4..1ac0b706ed091 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/abi.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
@@ -11,9 +11,9 @@ use rustc_target::callconv;
 use crate::IndexedVal;
 use crate::abi::{
     AddressSpace, ArgAbi, ArgAttributes, ArgExtension, CallConvention, CastTarget, FieldsShape,
-    FloatLength, FnAbi, IntegerLength, IntegerType, Layout, LayoutShape, NumScalableVectors,
-    PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar, TagEncoding, TyAndLayout,
-    Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange,
+    FloatLength, FnAbi, IndirectMode, IntegerLength, IntegerType, Layout, LayoutShape,
+    NumScalableVectors, PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar,
+    TagEncoding, TyAndLayout, Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange,
 };
 use crate::compiler_interface::BridgeTys;
 use crate::target::MachineSize as Size;
@@ -155,6 +155,22 @@ impl<'tcx> Stable<'tcx> for CanonAbi {
     }
 }
 
+impl<'tcx> Stable<'tcx> for callconv::IndirectMode {
+    type T = IndirectMode;
+
+    fn stable<'cx>(
+        &self,
+        _tables: &mut Tables<'cx, BridgeTys>,
+        _cx: &CompilerCtxt<'cx, BridgeTys>,
+    ) -> Self::T {
+        match self {
+            callconv::IndirectMode::Pointer => IndirectMode::Pointer,
+            callconv::IndirectMode::OnStack => IndirectMode::OnStack,
+            callconv::IndirectMode::AmdgpuKernelArg => IndirectMode::AmdgpuKernelArg,
+        }
+    }
+}
+
 impl<'tcx> Stable<'tcx> for callconv::PassMode {
     type T = PassMode;
 
@@ -172,11 +188,14 @@ impl<'tcx> Stable<'tcx> for callconv::PassMode {
             callconv::PassMode::Cast { pad_i32_count, cast } => {
                 PassMode::Cast { pad_i32_count: *pad_i32_count, cast: cast.stable(tables, cx) }
             }
-            callconv::PassMode::Indirect { attrs, meta_attrs, on_stack } => PassMode::Indirect {
-                attrs: attrs.stable(tables, cx),
-                meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)),
-                on_stack: *on_stack,
-            },
+            callconv::PassMode::Indirect { attrs, meta_attrs, address_space, mode } => {
+                PassMode::Indirect {
+                    attrs: attrs.stable(tables, cx),
+                    meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)),
+                    address_space: address_space.stable(tables, cx),
+                    mode: mode.stable(tables, cx),
+                }
+            }
         }
     }
 }
diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs
index 1ac6168035085..edc23b6c50b45 100644
--- a/compiler/rustc_target/src/callconv/mod.rs
+++ b/compiler/rustc_target/src/callconv/mod.rs
@@ -36,6 +36,25 @@ mod x86_win32;
 mod x86_win64;
 mod xtensa;
 
+/// Different modes in which indirect arguments can be passed.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, StableHash)]
+pub enum IndirectMode {
+    /// Passed as a normal pointer, nothing special.
+    Pointer,
+    /// The value should be passed at a fixed stack offset in accordance to
+    /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
+    /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
+    /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
+    /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
+    /// alignment (if `None`). This means that the alignment will not always
+    /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
+    OnStack,
+    /// `AmdgpuKernelArg` behaves similar to `OnStack` except that the pointer does not necessarily
+    /// point to the stack, no extra copy is made, and the passed argument should not be modified.
+    /// This corresponds to the `byref` LLVM argument attribute.
+    AmdgpuKernelArg,
+}
+
 #[derive(Clone, PartialEq, Eq, Hash, Debug, StableHash)]
 pub enum PassMode {
     /// Ignore the argument.
@@ -63,16 +82,17 @@ pub enum PassMode {
     /// The `meta_attrs` value, if any, is for the metadata (vtable or length) of an unsized
     /// argument. (This is the only mode that supports unsized arguments.)
     ///
-    /// `on_stack` defines that the value should be passed at a fixed stack offset in accordance to
-    /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
-    /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
-    /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
-    /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
-    /// alignment (if `None`). This means that the alignment will not always
-    /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
+    /// `address_space` specifies if the pointer is in a special address space or the default one.
     ///
-    /// `on_stack` cannot be true for unsized arguments, i.e., when `meta_attrs` is `Some`.
-    Indirect { attrs: ArgAttributes, meta_attrs: Option<ArgAttributes>, on_stack: bool },
+    /// `mode` can be a special way to pass an argument indirectly.
+    /// `OnStack` and `AmdgpuKernelArg` cannot be used for unsized arguments, i.e., when
+    /// `meta_attrs` is `Some`.
+    Indirect {
+        attrs: ArgAttributes,
+        meta_attrs: Option<ArgAttributes>,
+        address_space: Option<AddressSpace>,
+        mode: IndirectMode,
+    },
 }
 
 impl PassMode {
@@ -89,13 +109,23 @@ impl PassMode {
                 PassMode::Cast { cast: c2, pad_i32_count: pad2 },
             ) => c1.eq_abi(c2) && pad1 == pad2,
             (
-                PassMode::Indirect { attrs: a1, meta_attrs: None, on_stack: s1 },
-                PassMode::Indirect { attrs: a2, meta_attrs: None, on_stack: s2 },
-            ) => a1.eq_abi(a2) && s1 == s2,
+                PassMode::Indirect { attrs: a1, meta_attrs: None, address_space: as1, mode: m1 },
+                PassMode::Indirect { attrs: a2, meta_attrs: None, address_space: as2, mode: m2 },
+            ) => a1.eq_abi(a2) && as1 == as2 && m1 == m2,
             (
-                PassMode::Indirect { attrs: a1, meta_attrs: Some(e1), on_stack: s1 },
-                PassMode::Indirect { attrs: a2, meta_attrs: Some(e2), on_stack: s2 },
-            ) => a1.eq_abi(a2) && e1.eq_abi(e2) && s1 == s2,
+                PassMode::Indirect {
+                    attrs: a1,
+                    meta_attrs: Some(e1),
+                    address_space: as1,
+                    mode: m1,
+                },
+                PassMode::Indirect {
+                    attrs: a2,
+                    meta_attrs: Some(e2),
+                    address_space: as2,
+                    mode: m2,
+                },
+            ) => a1.eq_abi(a2) && as1 == as2 && e1.eq_abi(e2) && m1 == m2,
             _ => false,
         }
     }
@@ -424,7 +454,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
 
         let meta_attrs = layout.is_unsized().then_some(ArgAttributes::new());
 
-        PassMode::Indirect { attrs, meta_attrs, on_stack: false }
+        PassMode::Indirect { attrs, meta_attrs, address_space: None, mode: IndirectMode::Pointer }
     }
 
     /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
@@ -435,13 +465,31 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
             PassMode::Direct(_) | PassMode::Pair(_, _) => {
                 self.mode = Self::indirect_pass_mode(&self.layout);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => {
+            PassMode::Indirect {
+                attrs: _,
+                meta_attrs: _,
+                address_space: _,
+                mode: IndirectMode::Pointer,
+            } => {
                 // already indirect
             }
             _ => panic!("Tried to make {:?} indirect", self.mode),
         }
     }
 
+    /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
+    /// This is valid for both sized and unsized arguments.
+    #[track_caller]
+    pub fn make_indirect_addrspace(&mut self, addrspace: AddressSpace) {
+        self.make_indirect();
+        match self.mode {
+            PassMode::Indirect { ref mut address_space, .. } => {
+                *address_space = Some(addrspace);
+            }
+            _ => unreachable!(),
+        }
+    }
+
     /// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass
     /// ZSTs indirectly.
     #[track_caller]
@@ -450,7 +498,12 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
             PassMode::Ignore => {
                 self.mode = Self::indirect_pass_mode(&self.layout);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => {
+            PassMode::Indirect {
+                attrs: _,
+                meta_attrs: _,
+                address_space: _,
+                mode: IndirectMode::Pointer,
+            } => {
                 // already indirect
             }
             _ => panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode),
@@ -477,8 +530,8 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout");
         self.make_indirect();
         match self.mode {
-            PassMode::Indirect { ref mut attrs, meta_attrs: _, ref mut on_stack } => {
-                *on_stack = true;
+            PassMode::Indirect { ref mut attrs, meta_attrs: _, address_space: _, ref mut mode } => {
+                *mode = IndirectMode::OnStack;
 
                 // Some platforms, like 32-bit x86, change the alignment of the type when passing
                 // `byval`. Account for that.
@@ -492,6 +545,22 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         }
     }
 
+    /// Pass this argument indirectly.
+    /// This corresponds to the `byref` LLVM argument attribute.
+    ///
+    /// `address_space` specifies the address space of the passed pointer.
+    pub fn pass_amdgpu_kernel_arg(&mut self, addrspace: Option<AddressSpace>) {
+        assert!(!self.layout.is_unsized(), "used amdgpu kernel arg ABI for unsized layout");
+        self.make_indirect();
+        match self.mode {
+            PassMode::Indirect { attrs: _, meta_attrs: _, ref mut address_space, ref mut mode } => {
+                *mode = IndirectMode::AmdgpuKernelArg;
+                *address_space = addrspace;
+            }
+            _ => unreachable!(),
+        }
+    }
+
     pub fn extend_integer_width_to(&mut self, bits: u64) {
         // Only integers have signedness
         if let BackendRepr::Scalar(scalar) = self.layout.backend_repr
@@ -545,11 +614,17 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
     }
 
     pub fn is_sized_indirect(&self) -> bool {
-        matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ })
+        matches!(
+            self.mode,
+            PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ }
+        )
     }
 
     pub fn is_unsized_indirect(&self) -> bool {
-        matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ })
+        matches!(
+            self.mode,
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ }
+        )
     }
 
     pub fn is_ignore(&self) -> bool {
@@ -851,7 +926,7 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                     // Compute `Aggregate` ABI.
 
                     let is_indirect_not_on_stack =
-                        matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
+                        matches!(arg.mode, PassMode::Indirect { mode: IndirectMode::Pointer, .. });
                     assert!(is_indirect_not_on_stack);
 
                     let size = arg.layout.size;
@@ -962,7 +1037,7 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(ArgAbi<'_, usize>, 56);
-    static_assert_size!(FnAbi<'_, usize>, 80);
+    static_assert_size!(ArgAbi<'_, usize>, 64);
+    static_assert_size!(FnAbi<'_, usize>, 88);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_target/src/callconv/x86.rs b/compiler/rustc_target/src/callconv/x86.rs
index a1c59d885b7fc..3476f41e3bc75 100644
--- a/compiler/rustc_target/src/callconv/x86.rs
+++ b/compiler/rustc_target/src/callconv/x86.rs
@@ -200,12 +200,13 @@ pub(crate) fn fill_inregs<'a, Ty, C>(
 
     for arg in fn_abi.args.iter_mut() {
         let attrs = match arg.mode {
-            PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+            PassMode::Ignore
+            | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
                 continue;
             }
             PassMode::Direct(ref mut attrs) => attrs,
             PassMode::Pair(..)
-            | PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ }
+            | PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ }
             | PassMode::Cast { .. } => {
                 unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
             }
diff --git a/compiler/rustc_target/src/callconv/xtensa.rs b/compiler/rustc_target/src/callconv/xtensa.rs
index 4dc9fad650636..49005adeb33c0 100644
--- a/compiler/rustc_target/src/callconv/xtensa.rs
+++ b/compiler/rustc_target/src/callconv/xtensa.rs
@@ -7,7 +7,7 @@
 
 use rustc_abi::{BackendRepr, HasDataLayout, Size, TyAbiInterface};
 
-use crate::callconv::{ArgAbi, FnAbi, Reg, Uniform};
+use crate::callconv::{ArgAbi, FnAbi, IndirectMode, Reg, Uniform};
 use crate::spec::HasTargetSpec;
 
 const NUM_ARG_GPRS: u64 = 6;
@@ -29,8 +29,8 @@ where
     classify_arg_ty(cx, arg, &mut arg_gprs_left, true);
     // Ret args cannot be passed via stack, we lower to indirect and let the backend handle the invisible reference
     match arg.mode {
-        super::PassMode::Indirect { attrs: _, meta_attrs: _, ref mut on_stack } => {
-            *on_stack = false;
+        super::PassMode::Indirect { attrs: _, meta_attrs: _, address_space: _, ref mut mode } => {
+            *mode = IndirectMode::Pointer;
         }
         _ => {}
     }
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index bcba5839926db..a9d8aa23b1ea5 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -12,7 +12,9 @@ use rustc_middle::ty::layout::{
 use rustc_middle::ty::{self, InstanceKind, ShimKind, Ty, TyCtxt, Unnormalized};
 use rustc_span::def_id::DefId;
 use rustc_span::{DUMMY_SP, bug};
-use rustc_target::callconv::{AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, PassMode};
+use rustc_target::callconv::{
+    AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, IndirectMode, PassMode,
+};
 use tracing::debug;
 
 pub(crate) fn provide(providers: &mut Providers) {
@@ -444,15 +446,15 @@ fn fn_abi_sanity_check<'tcx>(
                 // omitted entirely in the calling convention.
                 assert!(arg.is_ignore());
             }
-            if let PassMode::Indirect { on_stack, .. } = arg.mode
+            if let PassMode::Indirect { mode, .. } = arg.mode
                 && spec_abi != ExternAbi::RustTail
             {
-                assert!(!on_stack, "rustic abi {spec_abi:?} shouldn't use on_stack");
+                assert!(mode == IndirectMode::Pointer, "rust abi must use plain pointer mode");
             }
         } else if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
             assert_matches!(
                 arg.mode,
-                PassMode::Indirect { on_stack: false, .. },
+                PassMode::Indirect { mode: IndirectMode::Pointer, .. },
                 "the {spec_abi} ABI does not implement `#[rustc_pass_indirectly_in_non_rustic_abis]`"
             );
         }
@@ -506,9 +508,9 @@ fn fn_abi_sanity_check<'tcx>(
                 // Indirect returns are arguments from an ABI perspective.
                 fn_arg_attrs_sanity_check(attrs, false);
             }
-            PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, on_stack } => {
+            PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, address_space: _, mode } => {
                 // With metadata. Must be unsized and not on the stack.
-                assert!(arg.layout.is_unsized() && !on_stack);
+                assert!(arg.layout.is_unsized() && *mode == IndirectMode::Pointer);
                 // Also, must not be `extern` type.
                 let tail = tcx.struct_tail_for_codegen(arg.layout.ty, cx.typing_env);
                 if matches!(tail.kind(), ty::Foreign(..)) {
diff --git a/tests/assembly-llvm/tail-call-indirect.rs b/tests/assembly-llvm/tail-call-indirect.rs
index 2bc1743a9bafd..918283966b405 100644
--- a/tests/assembly-llvm/tail-call-indirect.rs
+++ b/tests/assembly-llvm/tail-call-indirect.rs
@@ -10,10 +10,10 @@
 #![no_core]
 #![crate_type = "lib"]
 
-// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
 //
-// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
-// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to
+// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack
 // frame is overwritten by the callee's stack frame.
 //
 // The solution is to write the argument into the caller's argument place (stored somewhere further
diff --git a/tests/ui-fulldeps/rustc_public/check_abi.rs b/tests/ui-fulldeps/rustc_public/check_abi.rs
index ed616cc4d9bb0..92d6f28f8fd8c 100644
--- a/tests/ui-fulldeps/rustc_public/check_abi.rs
+++ b/tests/ui-fulldeps/rustc_public/check_abi.rs
@@ -15,8 +15,8 @@ extern crate rustc_middle;
 extern crate rustc_public;
 
 use rustc_public::abi::{
-    ArgAbi, ArgExtension, CallConvention, FieldsShape, IntegerLength, PassMode, Primitive, Scalar,
-    ValueRepr, VariantsShape,
+    ArgAbi, ArgExtension, CallConvention, FieldsShape, IndirectMode, IntegerLength, PassMode,
+    Primitive, Scalar, ValueRepr, VariantsShape,
 };
 use rustc_public::mir::MirVisitor;
 use rustc_public::mir::mono::Instance;
@@ -127,14 +127,14 @@ fn check_primitive(abi: &ArgAbi) {
 /// Check the return value: `Result<usize, &str>`.
 fn check_result(abi: &ArgAbi) {
     assert!(abi.ty.kind().is_enum());
-    let PassMode::Indirect { ref attrs, ref meta_attrs, on_stack } = abi.mode else {
+    let PassMode::Indirect { ref attrs, ref meta_attrs, address_space: _, mode } = abi.mode else {
         panic!("Expected PassMode::Indirect for Result, got: {:?}", abi.mode);
     };
     // Indirect arguments have a pointee alignment (the pointer must be aligned).
     assert!(attrs.pointee_align().is_some());
     // Result is a sized type, so no metadata pointer.
     assert!(meta_attrs.is_none());
-    assert!(!on_stack);
+    assert!(mode == IndirectMode::Pointer);
     let layout = abi.layout.shape();
     assert!(layout.is_sized());
     assert_matches!(layout.fields, FieldsShape::Arbitrary { .. });
diff --git a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
index 0bd4ac684066e..a54abdd5deeaf 100644
--- a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
+++ b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
@@ -23,7 +23,7 @@ use std::convert::TryFrom;
 use std::io::Write;
 use std::ops::ControlFlow;
 
-use rustc_public::abi::{CallConvention, PassMode, RegKind};
+use rustc_public::abi::{CallConvention, IndirectMode, PassMode, RegKind};
 use rustc_public::mir::mono::Instance;
 use rustc_public::{CrateDef, ItemKind};
 
@@ -147,7 +147,7 @@ fn test_abi_cast() -> ControlFlow<()> {
     }
     // Fourth TwoWords has no registers left → Indirect (on stack)
     assert!(
-        matches!(&abi.args[3].mode, PassMode::Indirect { on_stack: true, .. }),
+        matches!(&abi.args[3].mode, PassMode::Indirect { mode: IndirectMode::OnStack, .. }),
         "Expected arg 3 to be Indirect on stack, got: {:?}",
         abi.args[3].mode
     );
diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.powerpc-linux.stderr
+++ b/tests/ui/abi/c-zst.powerpc-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.s390x-linux.stderr
+++ b/tests/ui/abi/c-zst.s390x-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.sparc-linux.stderr b/tests/ui/abi/c-zst.sparc-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.sparc-linux.stderr
+++ b/tests/ui/abi/c-zst.sparc-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.sparc-none.stderr b/tests/ui/abi/c-zst.sparc-none.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.sparc-none.stderr
+++ b/tests/ui/abi/c-zst.sparc-none.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.sparc64-linux.stderr
+++ b/tests/ui/abi/c-zst.sparc64-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
+++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr
index 6242d93b09534..1793674fa462a 100644
--- a/tests/ui/abi/debug.generic.stderr
+++ b/tests/ui/abi/debug.generic.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr
index 176c68ecd4c7b..29ec7846101f1 100644
--- a/tests/ui/abi/debug.loongarch64.stderr
+++ b/tests/ui/abi/debug.loongarch64.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr
index 176c68ecd4c7b..29ec7846101f1 100644
--- a/tests/ui/abi/debug.riscv64.stderr
+++ b/tests/ui/abi/debug.riscv64.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/pass-indirectly-attr.rs b/tests/ui/abi/pass-indirectly-attr.rs
index 54aafc716587c..bb90b8354ea91 100644
--- a/tests/ui/abi/pass-indirectly-attr.rs
+++ b/tests/ui/abi/pass-indirectly-attr.rs
@@ -20,7 +20,7 @@ pub struct Type(u8);
 pub extern "C" fn extern_c(_: Type) {}
 //~^ ERROR fn_abi_of(extern_c) = FnAbi {
 //~| ERROR mode: Indirect
-//~| ERROR on_stack: false,
+//~| ERROR mode: Pointer,
 //~| ERROR conv: C,
 
 #[rustc_abi(debug)]
diff --git a/tests/ui/abi/pass-indirectly-attr.stderr b/tests/ui/abi/pass-indirectly-attr.stderr
index efeec0d86982b..5821e6279bb85 100644
--- a/tests/ui/abi/pass-indirectly-attr.stderr
+++ b/tests/ui/abi/pass-indirectly-attr.stderr
@@ -48,7 +48,8 @@ error: fn_abi_of(extern_c) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
index 45edd7bc0e0ee..c9e77ac941901 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
+++ b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/c-variadic/pass-by-value-abi.rs b/tests/ui/c-variadic/pass-by-value-abi.rs
index bcca09e90438a..317840601c050 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.rs
+++ b/tests/ui/c-variadic/pass-by-value-abi.rs
@@ -27,9 +27,9 @@ use std::ffi::VaList;
 pub extern "C" fn take_va_list(_: VaList<'_>) {}
 //~^ ERROR fn_abi_of(take_va_list) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
 //[aarch64]~^^^^ ERROR mode: Indirect {
-//[aarch64]~^^^^^ ERROR on_stack: false,
+//[aarch64]~^^^^^ ERROR mode: Pointer,
 //[win]~^^^^^^ ERROR mode: Direct(
 
 #[cfg(all(target_arch = "x86_64", not(windows)))]
@@ -37,11 +37,11 @@ pub extern "C" fn take_va_list(_: VaList<'_>) {}
 pub extern "sysv64" fn take_va_list_sysv64(_: VaList<'_>) {}
 //[x86_64]~^ ERROR fn_abi_of(take_va_list_sysv64) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
 
 #[cfg(all(target_arch = "x86_64", not(windows)))]
 #[rustc_abi(debug)]
 pub extern "win64" fn take_va_list_win64(_: VaList<'_>) {}
 //[x86_64]~^ ERROR: fn_abi_of(take_va_list_win64) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
diff --git a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
index 1e203b93e66b3..04320a5312361 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
+++ b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -113,7 +114,8 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -193,7 +195,8 @@ error: fn_abi_of(take_va_list_win64) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/explicit-tail-calls/indirect.rs b/tests/ui/explicit-tail-calls/indirect.rs
index b3e2613efad25..71107ef420c35 100644
--- a/tests/ui/explicit-tail-calls/indirect.rs
+++ b/tests/ui/explicit-tail-calls/indirect.rs
@@ -25,17 +25,17 @@
 #![feature(explicit_tail_calls)]
 #![expect(incomplete_features)]
 
-// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
 //
-// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
-// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to
+// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack
 // frame is overwritten by the callee's stack frame.
 //
 // The solution is to write the argument into the caller's argument place (stored somewhere further
 // up the stack), and forward that place.
 
 // A struct big enough that it is not passed via registers, so that the rust calling convention uses
-// `Indirect { on_stack: false, .. }`.
+// `Indirect { mode: IndirectMode::Pointer, .. }`.
 #[repr(C)]
 #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Big([u64; 4]);
@@ -79,7 +79,7 @@ fn main() {
     assert_eq!(update_in_caller(Big::default()), 0 + 2 + 3 + 4);
 
     assert_eq!(swapper(u8::MIN, u8::MAX), (u8::MAX, u8::MIN));
-    // i128 uses `PassMode::Indirect { on_stack: false, .. }` on x86_64 MSVC.
+    // i128 uses `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` on x86_64 MSVC.
     assert_eq!(swapper(i128::MIN, i128::MAX), (i128::MAX, i128::MIN));
     assert_eq!(swapper(Big([1; 4]), Big([2; 4])), (Big([2; 4]), Big([1; 4])));
 

From b8c747f548ba53a20b57484fc06b567a82aa227b Mon Sep 17 00:00:00 2001
From: Flakebi <flakebi@t-online.de>
Date: Thu, 3 Sep 2026 09:21:39 +0200
Subject: [PATCH 10/11] Pre-commit amdgpu gpu-kernel ABI test

---
 tests/codegen-llvm/amdgpu-abi/struct-abi.rs | 133 ++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 tests/codegen-llvm/amdgpu-abi/struct-abi.rs

diff --git a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
new file mode 100644
index 0000000000000..bf51cbfa7f7a0
--- /dev/null
+++ b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
@@ -0,0 +1,133 @@
+//@ add-minicore
+//@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 -Copt-level=3
+//@ needs-llvm-components: amdgpu
+#![feature(no_core, abi_gpu_kernel)]
+#![no_core]
+#![allow(improper_gpu_kernel_arg)]
+
+extern crate minicore;
+use minicore::num::Complex;
+
+// Tests from llvm-project/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
+
+#[repr(C)]
+pub struct SingleElementStructArg {
+    i: i32,
+}
+
+#[repr(C)]
+pub struct NestedSingleElementStructArg {
+    i: SingleElementStructArg,
+}
+
+#[repr(C)]
+pub struct StructArg {
+    i1: i32,
+    f: f32,
+    i2: i32,
+}
+
+#[repr(C)]
+pub struct StructPaddingArg {
+    i1: i8,
+    f: i64,
+}
+
+#[repr(C)]
+pub struct StructOfArraysArg {
+    i1: [i32; 2],
+    f1: f32,
+    i2: [i32; 4],
+    f2: [f32; 3],
+    i3: i32,
+}
+
+#[repr(C)]
+pub struct StructOfStructsArg {
+    i1: i32,
+    f1: f32,
+    s1: StructArg,
+    i2: i32,
+}
+
+#[repr(C)]
+pub union U {
+    b1: i32,
+    b2: f32,
+}
+
+#[repr(C)]
+pub struct SingleArrayElementStructArg {
+    i: [i32; 4],
+}
+
+#[repr(C)]
+pub struct SingleStructElementStructArgInner {
+    i: i32,
+    b: i64,
+}
+
+#[repr(C)]
+pub struct SingleStructElementStructArg {
+    s: SingleStructElementStructArgInner,
+}
+
+#[repr(C)]
+pub struct DifferentSizeTypePair {
+    l: i64,
+    i: i32,
+}
+
+// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_nested_single_element_struct_arg(
+    _: NestedSingleElementStructArg,
+) {
+}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(12) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_arg(_: StructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(i8 noundef {{%.+}}, i64 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_padding_arg(_: StructPaddingArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(44) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_of_arrays_arg(_: StructOfArraysArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(24) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_of_structs_arg(_: StructOfStructsArg) {}
+
+// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn test_kernel_union_arg(_: U) {}
+
+// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(16) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_array_element_struct_arg(_: SingleArrayElementStructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(i32 noundef {{%.+}}, i64 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_struct_element_struct_arg(
+    _: SingleStructElementStructArg,
+) {
+}
+
+// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(i64 noundef {{%.+}}, i32 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_different_size_type_pair_arg(_: DifferentSizeTypePair) {}
+
+// CHECK: define amdgpu_kernel void @kernel_complex(float noundef {{%.+}}, float noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_complex(_: Complex<f32>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_slice(ptr noalias nofree noundef nonnull readonly align 4 captures(none) {{%.+}}, i64 noundef range(i64 0, 2305843009213693952) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_slice(_: &[u32]) {}

From be988e9f8563e138c777cd4530594972a81af9b9 Mon Sep 17 00:00:00 2001
From: Flakebi <flakebi@t-online.de>
Date: Tue, 15 Sep 2026 10:33:14 +0200
Subject: [PATCH 11/11] Properly implement the gpu-kernel ABI for amdgpu
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add support to pass structs, arrays and vectors to amdgpu kernels.
Scalars and vectors are taken by value, aggregates are passed by byref
pointers. Structs containing a single scalar/vector are handled like
a scalar.

Judging from clang tests, nvptx seems to do somewhat the same, just
using byval instead of byref: https://github.com/llvm/llvm-project/blob/3a8affeef4da19d39191aac316e189eca3214a8c/clang/test/CodeGenCUDA/kernel-args.cu

I tested a couple of the lit test signatures on real hardware and it
seems to work fine. Given the relatively simple implementation, I hope
this amount of testing is enough (the C calling convention seems like
a worse fit for Rust’s current ABI code, it’s still giving me headaches).
---
 compiler/rustc_abi/src/lib.rs                |  4 +
 compiler/rustc_target/src/callconv/amdgpu.rs | 82 ++++++++++++++----
 tests/codegen-llvm/amdgpu-abi/struct-abi.rs  | 88 ++++++++++++++++----
 3 files changed, 139 insertions(+), 35 deletions(-)

diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 9c11405e9bb58..56cfbf138af00 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1778,6 +1778,10 @@ pub struct AddressSpace(pub u32);
 impl AddressSpace {
     /// LLVM's `0` address space.
     pub const ZERO: Self = AddressSpace(0);
+    /// The address space for constant memory on nvptx and amdgpu.
+    /// This address space is used e.g. for kernel arguments that are constant throughout the
+    /// execution.
+    pub const GPU_CONSTANT: Self = AddressSpace(4);
     /// The address space for workgroup memory on nvptx and amdgpu.
     /// See e.g. the `gpu_launch_sized_workgroup_mem` intrinsic for details.
     pub const GPU_WORKGROUP: Self = AddressSpace(3);
diff --git a/compiler/rustc_target/src/callconv/amdgpu.rs b/compiler/rustc_target/src/callconv/amdgpu.rs
index 98ab3ce8eb746..7a9eeaba19c96 100644
--- a/compiler/rustc_target/src/callconv/amdgpu.rs
+++ b/compiler/rustc_target/src/callconv/amdgpu.rs
@@ -1,25 +1,60 @@
-use rustc_abi::{HasDataLayout, TyAbiInterface};
+use rustc_abi::{
+    AddressSpace, BackendRepr, CanonAbi, HasDataLayout, Reg, RegKind, TyAbiInterface, TyAndLayout,
+};
 
-use crate::callconv::{ArgAbi, FnAbi};
+use crate::callconv::{FnAbi, Uniform};
 
-fn classify_ret<'a, Ty, C>(_cx: &C, ret: &mut ArgAbi<'a, Ty>)
-where
-    Ty: TyAbiInterface<'a, C> + Copy,
-    C: HasDataLayout,
-{
-    ret.extend_integer_width_to(32);
-}
+// For reference, see llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp
 
-fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
+/// If the given type is a (potentially nested) struct containing a single scalar, return
+/// a `Uniform` for the contained, single element.
+fn single_element_struct_to_reg<'a, Ty, C>(cx: &C, ty: TyAndLayout<'a, Ty>) -> Option<Uniform>
 where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
-        arg.make_indirect();
-        return;
+    assert!(ty.is_aggregate(), "Only handles aggregate types");
+    if ty.layout.fields.count() != 1 {
+        return None;
+    }
+    let field = ty.field(cx, 0);
+    match field.backend_repr {
+        BackendRepr::SimdScalableVector { .. } => panic!("scalable vectors are unsupported"),
+        BackendRepr::Scalar(_) => {
+            // Check that the size is the same as the size for ty, so no extra padding
+            let size = field.layout.size.bytes();
+            if ty.layout.size.bytes() != size {
+                return None;
+            }
+
+            // clang passes the inner type directly, we emulate it with fitting integer types
+            match size {
+                1 => Some(Uniform::new(Reg::i8(), field.layout.size)),
+                2 => Some(Uniform::new(Reg::i16(), field.layout.size)),
+                4 => Some(Uniform::new(Reg::i32(), field.layout.size)),
+                8 => Some(Uniform::new(Reg::i64(), field.layout.size)),
+                16 => Some(Uniform::new(Reg::i128(), field.layout.size)),
+                s => panic!("Unhandled scalar of size {s} in amdgpu gpu-kernel ABI"),
+            }
+        }
+        BackendRepr::SimdVector { element, .. } => {
+            // Check that the size is the same as the size for ty, so no extra padding
+            let size = field.layout.size.bytes();
+            if ty.layout.size.bytes() != size {
+                return None;
+            }
+
+            // clang passes the inner type directly, we emulate it with a vector of the same type.
+            // The size is rounded up to the size of the complete type (including alignment).
+            let reg = Reg {
+                kind: RegKind::Vector { hint_vector_elem: element.primitive() },
+                size: field.layout.size,
+            };
+            Some(Uniform::new(reg, field.layout.size))
+        }
+        BackendRepr::Memory { .. } => single_element_struct_to_reg(cx, field),
+        BackendRepr::ScalarPair { .. } => None,
     }
-    arg.extend_integer_width_to(32);
 }
 
 pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
@@ -27,14 +62,25 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if !fn_abi.ret.is_ignore() {
-        classify_ret(cx, &mut fn_abi.ret);
-    }
+    // Kernels cannot return values, so do not handle return types
 
+    // Try to fill first registers with values and pass by_ref pointers for later indirect arguments
     for arg in fn_abi.args.iter_mut() {
         if arg.is_ignore() {
             continue;
         }
-        classify_arg(cx, arg);
+        if fn_abi.conv == CanonAbi::GpuKernel {
+            if arg.layout.is_aggregate() {
+                if let Some(uniform) = single_element_struct_to_reg(cx, arg.layout) {
+                    // Single element structs are passed directly as the inner type
+                    arg.cast_to(uniform);
+                } else {
+                    // All other aggregates are passed as by_ref pointer in the constant address space
+                    arg.pass_amdgpu_kernel_arg(Some(AddressSpace::GPU_CONSTANT));
+                }
+            }
+        } else {
+            // FIXME: C ABI is not yet implemented
+        }
     }
 }
diff --git a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
index bf51cbfa7f7a0..bc83f6510a6fe 100644
--- a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
+++ b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
@@ -1,7 +1,7 @@
 //@ add-minicore
 //@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 -Copt-level=3
 //@ needs-llvm-components: amdgpu
-#![feature(no_core, abi_gpu_kernel)]
+#![feature(no_core, abi_gpu_kernel, repr_simd)]
 #![no_core]
 #![allow(improper_gpu_kernel_arg)]
 
@@ -10,14 +10,32 @@ use minicore::num::Complex;
 
 // Tests from llvm-project/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
 
+#[repr(simd)]
+pub struct I8X2([i8; 2]);
+
+#[repr(simd)]
+pub struct I16X2([i16; 2]);
+
+#[repr(simd)]
+pub struct I16X3([i16; 3]);
+
+#[repr(simd)]
+pub struct I16X4([i16; 4]);
+
+#[repr(simd)]
+pub struct I32X3([i32; 3]);
+
+#[repr(simd)]
+pub struct I32X4([i32; 4]);
+
 #[repr(C)]
-pub struct SingleElementStructArg {
-    i: i32,
+pub struct SingleElementStructArg<T> {
+    i: T,
 }
 
 #[repr(C)]
 pub struct NestedSingleElementStructArg {
-    i: SingleElementStructArg,
+    i: SingleElementStructArg<i32>,
 }
 
 #[repr(C)]
@@ -78,56 +96,92 @@ pub struct DifferentSizeTypePair {
     i: i32,
 }
 
-// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(i32 %0)
 #[no_mangle]
-pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg) {}
+pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg<i32>) {}
 
-// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(i32 %0)
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_nested_single_element_struct_arg(
     _: NestedSingleElementStructArg,
 ) {
 }
 
-// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(12) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([12 x i8]) align 4 captures(none) dereferenceable(12) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_arg(_: StructArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(i8 noundef {{%.+}}, i64 noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_padding_arg(_: StructPaddingArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(44) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr addrspace(4) noalias nofree noundef readnone byref([44 x i8]) align 4 captures(none) dereferenceable(44) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_of_arrays_arg(_: StructOfArraysArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(24) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr addrspace(4) noalias nofree noundef readnone byref([24 x i8]) align 4 captures(none) dereferenceable(24) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_of_structs_arg(_: StructOfStructsArg) {}
 
-// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr addrspace(4) noalias nofree noundef readnone byref([4 x i8]) align 4 captures(none) dereferenceable(4) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn test_kernel_union_arg(_: U) {}
 
-// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(16) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 4 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_single_array_element_struct_arg(_: SingleArrayElementStructArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(i32 noundef {{%.+}}, i64 noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_single_struct_element_struct_arg(
     _: SingleStructElementStructArg,
 ) {
 }
 
-// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(i64 noundef {{%.+}}, i32 noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_different_size_type_pair_arg(_: DifferentSizeTypePair) {}
 
-// CHECK: define amdgpu_kernel void @kernel_complex(float noundef {{%.+}}, float noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_complex(ptr addrspace(4) noalias nofree noundef readnone byref([8 x i8]) align 4 captures(none) dereferenceable(8) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_complex(_: Complex<f32>) {}
 
-// CHECK: define amdgpu_kernel void @kernel_slice(ptr noalias nofree noundef nonnull readonly align 4 captures(none) {{%.+}}, i64 noundef range(i64 0, 2305843009213693952) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_slice(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_slice(_: &[u32]) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i64(i64 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i64(_: i64) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i64_struct(i64 {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i64_struct(_: SingleElementStructArg<i64>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i128_struct(i128 {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i128_struct(_: SingleElementStructArg<i128>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i8x2_struct(<2 x i8> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i8x2_struct(_: SingleElementStructArg<I8X2>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x2_struct(<2 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x2_struct(_: SingleElementStructArg<I16X2>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x3_struct(<4 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x3_struct(_: SingleElementStructArg<I16X3>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x4_struct(<4 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x4_struct(_: SingleElementStructArg<I16X4>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i32x3_struct(<4 x i32> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i32x3_struct(_: SingleElementStructArg<I32X3>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i32x4_struct(<4 x i32> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i32x4_struct(_: SingleElementStructArg<I32X4>) {}