diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs
index e51255dc5963f..b8928aecf0cc5 100644
--- a/compiler/rustc_abi/src/layout/ty.rs
+++ b/compiler/rustc_abi/src/layout/ty.rs
@@ -233,8 +233,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
     }
 
     /// If this method returns `true`, then this type should always have a `PassMode` of
-    /// `Indirect { on_stack: false, .. }` when being used as the argument type of a function with a
-    /// non-Rustic ABI (this is true for structs annotated with the
+    /// `Indirect { mode: IndirectMode::Pointer, .. }` when being used as the argument type of a
+    /// function with a non-Rustic ABI (this is true for structs annotated with the
     /// `#[rustc_pass_indirectly_in_non_rustic_abis]` attribute).
     ///
     /// This is used to replicate some of the behaviour of C array-to-pointer decay; however unlike
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 9c11405e9bb58..56cfbf138af00 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1778,6 +1778,10 @@ pub struct AddressSpace(pub u32);
 impl AddressSpace {
     /// LLVM's `0` address space.
     pub const ZERO: Self = AddressSpace(0);
+    /// The address space for constant memory on nvptx and amdgpu.
+    /// This address space is used e.g. for kernel arguments that are constant throughout the
+    /// execution.
+    pub const GPU_CONSTANT: Self = AddressSpace(4);
     /// The address space for workgroup memory on nvptx and amdgpu.
     /// See e.g. the `gpu_launch_sized_workgroup_mem` intrinsic for details.
     pub const GPU_WORKGROUP: Self = AddressSpace(3);
diff --git a/compiler/rustc_attr_ir/src/attribute_docs.rs b/compiler/rustc_attr_ir/src/attribute_docs.rs
index 4d40cd6d52f16..a2b00c90b3028 100644
--- a/compiler/rustc_attr_ir/src/attribute_docs.rs
+++ b/compiler/rustc_attr_ir/src/attribute_docs.rs
@@ -223,3 +223,63 @@ const _: () = ();
 ///
 /// [`vtable_entries`]: ../rustc_middle/ty/struct.TyCtxt.html#method.vtable_entries
 const _: () = ();
+
+#[doc(attribute = "rustc_dyn_incompatible_trait", alias = "rustc_do_not_implement_via_object")]
+/// Opts a trait out of [dyn compatibility].
+///
+/// This is useful to reserve the ability to add dyn incompatible supertraits or methods to a trait
+/// in the future and to ensure the soundness of various constructs - see below for more about that.
+///
+/// For example [`Field`], [`FnPtr`], [`Tuple`], [`TransmuteFrom`], [`Sized`] and [`Unsize`] must
+/// be dyn incompatible because these traits describe properties and layouts of types that would
+/// be invalid for trait objects.
+///
+/// While making a trait dyn incompatible can also be done by including a (hidden and/or unstable)
+/// dyn incompatible method in the trait, using `#[rustc_dyn_incompatible_trait]` should be
+/// preferred because it is self-documenting and generates better error messages.
+///
+/// # Example
+///
+#[doc = include_example!("rustc_dyn_incompatible_trait")]
+///
+/// # Unsafe traits and dyn (in)compatibility
+///
+/// [Recall] that a trait object (`dyn Trait`) implements the base trait, its auto traits, and any supertraits of
+/// the base trait. This means that it's possible to run into subtle soundness problems when relying
+/// on the safety contract of a dyn compatible unsafe trait. See the following example:
+///
+// ignore-tidy-odd-backticks
+/// ```should_panic
+#[doc = include_str!("../../../tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs")]
+// ignore-tidy-odd-backticks
+/// ```
+///
+/// A solution for this is to make `UnsafeTrait` dyn incompatible, forcing `LocalTrait` to also be
+/// dyn incompatible so that a `dyn LocalTrait` cannot be formed. This is what we ended up doing for
+/// [#154619].
+///
+/// [`Allocator`] has had similar problems with  [`Clone`] ([#156920])
+/// but as we really wanted `dyn Allocator` to be a thing we ended up not making it dyn
+/// incompatible -- we ended up moving the safety contract to [`AllocatorClone`] instead.
+///
+/// See also [#156917] and [#160045] for more examples of this problem.
+///
+/// See [`AttributeKind::RustcDynIncompatibleTrait`] for the internal representation of this attribute.
+///
+/// [`Allocator`]: core::alloc::Allocator
+/// [`AllocatorClone`]: core::alloc::AllocatorClone
+/// [`Clone`]: core::clone::Clone
+/// [`Field`]: core::field::Field
+// FIXME: use core::ops::FnPtr once trickled down to beta
+/// [`FnPtr`]: https://doc.rust-lang.org/nightly/core/ops/trait.FnPtr.html
+/// [`Tuple`]: core::marker::Tuple
+/// [`TransmuteFrom`]: core::mem::TransmuteFrom
+/// [`Sized`]: core::marker::Sized
+/// [`Unsize`]: core::marker::Unsize
+/// [dyn compatibility]: https://doc.rust-lang.org/nightly/reference/items/traits.html#dyn-compatibility
+/// [recall]: https://doc.rust-lang.org/nightly/reference/types/trait-object.html#r-type.trait-object.impls
+/// [#154619]: https://github.com/rust-lang/rust/issues/154619 "`deref_patterns` is unsound due to `dyn` of subtrait of `DerefPure`"
+/// [#156917]: https://github.com/rust-lang/rust/issues/156917 "`dyn Allocator` together with `Allocator + PartialEq` safety requirements leads to unsoundness"
+/// [#156920]:https://github.com/rust-lang/rust/issues/156920 "`dyn Allocator` together with `Allocator + Clone` requirements is unsound, leading to UB with `Arc`"
+/// [#160045]:https://github.com/rust-lang/rust/issues/160045 "`iter::Rev`'s `TrustedLen` impl is unsound with trait objects"
+const _: () = ();
diff --git a/compiler/rustc_attr_ir/src/data_structures.rs b/compiler/rustc_attr_ir/src/data_structures.rs
index d722d515582dc..712ed41ae7759 100644
--- a/compiler/rustc_attr_ir/src/data_structures.rs
+++ b/compiler/rustc_attr_ir/src/data_structures.rs
@@ -1267,7 +1267,7 @@ pub enum AttributeKind {
     /// Represents the [`rustc_dump_vtable`](./attribute.rustc_dump_vtable.html) attribute.
     RustcDumpVtable(Span),
 
-    /// Represents `#[rustc_dyn_incompatible_trait]`.
+    /// Represents the [`rustc_dyn_incompatible_trait`](./attribute.rustc_dyn_incompatible_trait.html) attribute.
     RustcDynIncompatibleTrait(Span),
 
     /// Represents `#[rustc_effective_visibility]`.
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 1c552ca1a9c32..48ffc43c5cfa1 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -3,7 +3,7 @@
 use cranelift_codegen::ir::ArgumentPurpose;
 use rustc_abi::{Reg, RegKind};
 use rustc_target::callconv::{
-    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode,
+    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, IndirectMode, PassMode,
 };
 use smallvec::{SmallVec, smallvec};
 
@@ -126,8 +126,12 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 assert_eq!(pad_i32_count, 0, "padding support not yet implemented");
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
             }
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
-                if on_stack {
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
+                assert!(
+                    mode != IndirectMode::AmdgpuKernelArg,
+                    "unsupported amdgpu kernel argument"
+                );
+                if mode == IndirectMode::OnStack {
                     // Abi requires aligning struct size to pointer size
                     let size = self.layout.size.align_to(tcx.data_layout.pointer_align().abi);
                     let size = u32::try_from(size.bytes()).unwrap();
@@ -139,8 +143,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                     smallvec![apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)]
                 }
             }
-            PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), address_space: _, mode } => {
+                assert!(mode == IndirectMode::Pointer);
                 smallvec![
                     apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs),
                     apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs),
@@ -184,8 +188,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 None,
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(),
             ),
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
+                assert!(mode == IndirectMode::Pointer);
                 (
                     Some(apply_attrs_to_abi_param(
                         AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn),
@@ -194,7 +198,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                     vec![],
                 )
             }
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 unreachable!("unsized return value")
             }
         }
@@ -324,7 +328,7 @@ pub(super) fn cvalue_for_param<'tcx>(
         PassMode::Cast { ref cast, .. } => {
             from_casted_value(fx, &block_params, arg_abi.layout, cast)
         }
-        PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => {
             assert_eq!(block_params.len(), 1, "{:?}", block_params);
             if let Some(pointee_align) = attrs.pointee_align
                 && pointee_align < arg_abi.layout.align.abi
@@ -342,7 +346,7 @@ pub(super) fn cvalue_for_param<'tcx>(
                 CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
             CValue::by_ref_unsized(Pointer::new(block_params[0]), block_params[1], arg_abi.layout)
         }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index 36087f96dd776..7f4ee9435b506 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -17,12 +17,12 @@ pub(super) fn codegen_return_param<'tcx>(
             let is_ssa = ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.ret.layout.ty);
             (super::make_local_place(fx, RETURN_PLACE, fx.fn_abi.ret.layout, is_ssa), smallvec![])
         }
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             let ret_param = block_params_iter.next().unwrap();
             assert_eq!(fx.bcx.func.dfg.value_type(ret_param), fx.pointer_type);
             (CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.ret.layout), smallvec![ret_param])
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
     };
@@ -50,7 +50,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
 ) {
     let (ret_temp_place, return_ptr) = match ret_arg_abi.mode {
         PassMode::Ignore => (None, None),
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             if let Some(ret_ptr) = ret_place.try_to_ptr() {
                 // This is an optimization to prevent unnecessary copies of the return value when
                 // the return place is already a memory place as opposed to a register.
@@ -61,7 +61,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
                 (Some(place), Some(place.to_ptr().get_addr(fx)))
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None),
@@ -86,14 +86,14 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
                 super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast);
             ret_place.write_cvalue(fx, result);
         }
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             if let Some(ret_temp_place) = ret_temp_place {
                 // If ret_temp_place is None, it is not necessary to copy the return value.
                 let ret_temp_value = ret_temp_place.to_cvalue(fx);
                 ret_place.write_cvalue(fx, ret_temp_value);
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
     }
@@ -102,10 +102,11 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
 /// Codegen a return instruction with the right return value(s) if any.
 pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
     match fx.fn_abi.ret.mode {
-        PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Ignore
+        | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             fx.bcx.ins().return_(&[]);
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
         PassMode::Direct(_) => {
diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs
index 63eaf52ce9f01..5b88cebb4f174 100644
--- a/compiler/rustc_codegen_gcc/src/abi.rs
+++ b/compiler/rustc_codegen_gcc/src/abi.rs
@@ -11,7 +11,7 @@ use rustc_middle::ty::layout::LayoutOf;
 #[cfg(feature = "master")]
 use rustc_session::{Session, config};
 use rustc_span::bug;
-use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode};
 #[cfg(feature = "master")]
 use rustc_target::spec::Arch;
 
@@ -189,7 +189,12 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     let ty = cast.gcc_type(cx);
                     apply_attrs(ty, &cast.attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs: _,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let x86_interrupt_first_arg = {
                         #[cfg(feature = "master")]
                         {
@@ -216,14 +221,32 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         arg.layout.gcc_type(cx)
                     }
                 }
+                PassMode::Indirect {
+                    attrs: _,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    unimplemented!("unsupported amdgpu kernel argument")
+                }
                 PassMode::Direct(attrs) => {
                     apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     apply_attrs(cx.type_ptr_to(arg.layout.gcc_type(cx)), &attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                    assert!(!on_stack);
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode,
+                } => {
+                    assert!(mode == IndirectMode::Pointer);
                     // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
                     // Any two ABI-compatible unsized types have the same metadata type and
                     // moreover the same metadata value leads to the same dynamic size and
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index dce8db6841b7f..e16f34483a0c5 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -15,7 +15,7 @@ use rustc_middle::ty::layout::LayoutOf;
 use rustc_session::{Session, config};
 use rustc_span::bug;
 use rustc_target::callconv::{
-    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode,
+    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, IndirectMode, PassMode,
 };
 use rustc_target::spec::{Arch, SanitizerSet};
 use smallvec::SmallVec;
@@ -243,12 +243,12 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         match &self.mode {
             PassMode::Ignore => {}
             // Sized indirect arguments
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => {
                 let align = attrs.pointee_align.unwrap_or(self.layout.align.abi);
                 OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst);
             }
             // Unsized indirect arguments cannot be stored
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 bug!("unsized `ArgAbi` cannot be stored");
             }
             PassMode::Cast { cast, pad_i32_count: _ } => {
@@ -304,11 +304,11 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             PassMode::Pair(..) => {
                 OperandValue::Pair(next(), next()).store(bx, dst);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 bug!("unsized `ArgAbi` cannot be stored");
             }
             PassMode::Direct(_)
-            | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ }
+            | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ }
             | PassMode::Cast { .. } => {
                 let next_arg = next();
                 self.store(bx, next_arg, dst);
@@ -369,8 +369,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Ignore => cx.type_void(),
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
             PassMode::Cast { cast, pad_i32_count: _ } => cast.llvm_type(cx),
-            PassMode::Indirect { .. } => {
-                llargument_tys.push(cx.type_ptr());
+            PassMode::Indirect { address_space, .. } => {
+                let ty = if let Some(address_space) = address_space {
+                    cx.type_ptr_ext(*address_space)
+                } else {
+                    cx.type_ptr()
+                };
+                llargument_tys.push(ty);
                 cx.type_void()
             }
         };
@@ -395,7 +400,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                     // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
                     // Any two ABI-compatible unsized types have the same metadata type and
                     // moreover the same metadata value leads to the same dynamic size and
@@ -406,7 +411,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(),
+                PassMode::Indirect { attrs: _, meta_attrs: None, address_space, mode: _ } => {
+                    if let Some(address_space) = address_space {
+                        cx.type_ptr_ext(*address_space)
+                    } else {
+                        cx.type_ptr()
+                    }
+                }
                 PassMode::Cast { cast, pad_i32_count } => {
                     // Add padding.
                     llargument_tys.extend(std::iter::repeat_n(
@@ -496,8 +507,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     apply_range_attr(llvm::AttributePlace::ReturnValue, scalar);
                 }
             }
-            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => {
+                assert!(*mode == IndirectMode::Pointer);
                 let i = apply(attrs);
                 let sret = llvm::CreateStructRetAttr(
                     cx.llcx,
@@ -523,7 +534,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         for arg in self.args.iter() {
             match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let i = apply(attrs);
                     let byval = llvm::CreateByValAttr(
                         cx.llcx,
@@ -531,13 +547,31 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     );
                     attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
                 }
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    let i = apply(attrs);
+                    let byref = llvm::CreateByRefAttr(
+                        cx.llcx,
+                        cx.type_array(cx.type_i8(), arg.layout.size.bytes()),
+                    );
+                    attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byref]);
+                }
                 PassMode::Direct(attrs) => {
                     let i = apply(attrs);
                     if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
                         apply_range_attr(llvm::AttributePlace::Argument(i), scalar);
                     }
                 }
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     let i = apply(attrs);
                     if cx.sess().opts.optimize != config::OptLevel::No {
                         attributes::apply_to_llfn(
@@ -547,8 +581,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         );
                     }
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                    assert!(!on_stack);
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode,
+                } => {
+                    assert!(*mode == IndirectMode::Pointer);
                     apply(attrs);
                     apply(meta_attrs);
                 }
@@ -626,8 +665,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Direct(attrs) => {
                 attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite);
             }
-            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => {
+                assert!(*mode == IndirectMode::Pointer);
                 let i = apply(bx.cx, attrs);
                 let sret = llvm::CreateStructRetAttr(
                     bx.cx.llcx,
@@ -647,7 +686,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         for arg in self.args.iter() {
             match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let i = apply(bx.cx, attrs);
                     let byval = llvm::CreateByValAttr(
                         bx.cx.llcx,
@@ -659,11 +703,38 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         &[byval],
                     );
                 }
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    let i = apply(bx.cx, attrs);
+                    let byref = llvm::CreateByRefAttr(
+                        bx.cx.llcx,
+                        bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()),
+                    );
+                    attributes::apply_to_callsite(
+                        callsite,
+                        llvm::AttributePlace::Argument(i),
+                        &[byref],
+                    );
+                }
                 PassMode::Direct(attrs)
-                | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                | PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     apply(bx.cx, attrs);
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack: _ } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode: _,
+                } => {
                     apply(bx.cx, attrs);
                     apply(bx.cx, meta_attrs);
                 }
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index b3882ba51749d..dd2dbdff6b82c 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -825,9 +825,7 @@ pub(crate) unsafe fn llvm_optimize(
 
     // This assumes that we previously compiled our kernels for a gpu target, which created a
     // `device.bin` artifact. The user is supposed to provide us with a path to this artifact, we
-    // don't need any other artifacts from the previous run. We will embed this artifact into our
-    // LLVM-IR host module, to create a `host.o` ObjectFile, which we will write to disk.
-    // The last, not yet automated steps uses the `clang-linker-wrapper` to process `host.o`.
+    // don't need any other artifacts from the previous run.
     if !cgcx.target_is_like_gpu && is_final_stage {
         if let Some(device_path) = config
             .offload
@@ -846,38 +844,7 @@ pub(crate) unsafe fn llvm_optimize(
             } else if !device_pathbuf.exists() {
                 dcx.emit_err(crate::diagnostics::OffloadNonexistingPath);
             }
-            let host_path = cgcx.output_filenames.path(OutputType::Object);
-            let host_dir = host_path.parent().unwrap();
-            let out_obj = host_dir.join("host.o");
             let device_bin_c = path_to_c_string(device_pathbuf.as_path());
-
-            // 2) Finalize host: lib.bc + device.bin -> host.o (host TM)
-            // We create a full clone of our LLVM host module, since we will embed the device IR
-            // into it, and this might break caching or incremental compilation otherwise.
-            let ok = unsafe {
-                llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_embed_buffer_in_module(
-                    module.module_llvm.llmod(),
-                    device_bin_c.as_c_str(),
-                )
-            };
-            if !ok {
-                dcx.emit_err(crate::diagnostics::OffloadEmbedFailed);
-            }
-            write_output_file(
-                dcx,
-                module.module_llvm.tm.raw(),
-                config.no_builtins,
-                module.module_llvm.llmod(),
-                &out_obj,
-                None,
-                llvm::FileType::ObjectFile,
-                prof,
-                true,
-            );
-            // We ignore cgcx.save_temps here and unconditionally always keep our `device.bin` artifact.
-            // Otherwise, recompiling the host code would fail since we deleted that device artifact
-            // in the previous host compilation, which would be confusing at best.
-
             let ok = unsafe {
                 llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_wrap_images(
                     module.module_llvm.llmod(),
diff --git a/compiler/rustc_codegen_llvm/src/diagnostics.rs b/compiler/rustc_codegen_llvm/src/diagnostics.rs
index 6c5c23203aa79..08b96770e46d1 100644
--- a/compiler/rustc_codegen_llvm/src/diagnostics.rs
+++ b/compiler/rustc_codegen_llvm/src/diagnostics.rs
@@ -101,10 +101,6 @@ pub(crate) struct OffloadNonexistingPath;
 #[diag("call to BundleImages failed, `device.bin` was not created")]
 pub(crate) struct OffloadBundleImagesFailed;
 
-#[derive(Diagnostic)]
-#[diag("call to EmbedBufferInModule failed, `host.o` was not created")]
-pub(crate) struct OffloadEmbedFailed;
-
 #[derive(Diagnostic)]
 #[diag("call to WrapImages failed, device image was not wrapped into the host module")]
 pub(crate) struct OffloadWrapImagesFailed;
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 52218bfa336b8..07381745afe84 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -2016,6 +2016,7 @@ unsafe extern "C" {
     pub(crate) fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute;
     pub(crate) fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute;
     pub(crate) fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
+    pub(crate) fn LLVMRustCreateByRefAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute;
diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
index 5452f4abc5c33..89e4d60656d34 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
@@ -122,6 +122,10 @@ pub(crate) fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll At
     unsafe { LLVMRustCreateByValAttr(llcx, ty) }
 }
 
+pub(crate) fn CreateByRefAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute {
+    unsafe { LLVMRustCreateByRefAttr(llcx, ty) }
+}
+
 pub(crate) fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute {
     unsafe { LLVMRustCreateStructRetAttr(llcx, ty) }
 }
diff --git a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs
index fecb9e40eab88..da3da6bd7abd4 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs
@@ -5,7 +5,6 @@ use std::sync::OnceLock;
 use super::ffi::{Module, TargetMachine, Value};
 
 type LLVMRustBundleImagesFn = unsafe extern "C" fn(&Module, &TargetMachine, *const c_char) -> bool;
-type LLVMRustOffloadEmbedBufferInModuleFn = unsafe extern "C" fn(&Module, *const c_char) -> bool;
 type LLVMRustOffloadMapperFn = unsafe extern "C" fn(&Value, &Value, *const &Value);
 type LLVMRustOffloadWrapImagesFn =
     unsafe extern "C" fn(&Module, *const c_char, *const c_char) -> bool;
@@ -18,7 +17,6 @@ use crate::llvm;
 
 pub(crate) struct RustOffloadWrapper {
     LLVMRustBundleImages: LLVMRustBundleImagesFn,
-    LLVMRustOffloadEmbedBufferInModule: LLVMRustOffloadEmbedBufferInModuleFn,
     LLVMRustOffloadMapper: LLVMRustOffloadMapperFn,
     LLVMRustOffloadWrapImages: LLVMRustOffloadWrapImagesFn,
     lld_path: Option<PathBuf>,
@@ -65,14 +63,6 @@ impl RustOffloadWrapper {
         unsafe { (self.LLVMRustBundleImages)(m, tm, c.as_ptr()) }
     }
 
-    pub(crate) unsafe fn llvm_rust_offload_embed_buffer_in_module(
-        &self,
-        m: &Module,
-        i: &CStr,
-    ) -> bool {
-        unsafe { (self.LLVMRustOffloadEmbedBufferInModule)(m, i.as_ptr()) }
-    }
-
     pub(crate) unsafe fn llvm_rust_offload_wrapper(&self, v1: &Value, v2: &Value, vs: &[&Value]) {
         unsafe { (self.LLVMRustOffloadMapper)(v1, v2, vs.as_ptr()) }
     }
@@ -96,11 +86,6 @@ impl RustOffloadWrapper {
 
         let llvm_rust_bundle_images =
             *unsafe { lib.get::<LLVMRustBundleImagesFn>(b"LLVMRustBundleImages\0")? };
-        let llvm_rust_offload_embed_buffer_in_module = *unsafe {
-            lib.get::<LLVMRustOffloadEmbedBufferInModuleFn>(
-                b"LLVMRustOffloadEmbedBufferInModule\0",
-            )?
-        };
         let llvm_rust_offload_wrapper =
             *unsafe { lib.get::<LLVMRustOffloadMapperFn>(b"LLVMRustOffloadMapper\0")? };
         let llvm_rust_offload_wrap_images =
@@ -108,7 +93,6 @@ impl RustOffloadWrapper {
 
         Ok(Self {
             LLVMRustBundleImages: llvm_rust_bundle_images,
-            LLVMRustOffloadEmbedBufferInModule: llvm_rust_offload_embed_buffer_in_module,
             LLVMRustOffloadMapper: llvm_rust_offload_wrapper,
             LLVMRustOffloadWrapImages: llvm_rust_offload_wrap_images,
             lld_path,
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 6b0def4ffa182..f99009a0f4243 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -18,7 +18,7 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
 use rustc_session::config::OptLevel;
 use rustc_span::{Span, Spanned, bug, span_bug};
-use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode};
 use tracing::{debug, info};
 
 use super::operand::OperandRef;
@@ -1257,7 +1257,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             (args, None)
         };
 
-        // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+        // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
         //
         // Normally an indirect argument that is allocated in the caller's stack frame
         // would be passed as a pointer into the callee's stack frame.
@@ -1282,10 +1282,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let mut tail_call_temporaries = vec![];
         if kind == CallKind::Tail {
             tail_call_temporaries = vec![None; first_args.len()];
-            // Copy the arguments that use `PassMode::Indirect { on_stack: false , ..}`
+            // Copy the arguments that use `PassMode::Indirect { mode: IndirectMode::Pointer , ..}`
             // to temporary stack allocations. See the comment above.
             for (i, arg) in first_args.iter().enumerate() {
-                if !matches!(fn_abi.args[i].mode, PassMode::Indirect { on_stack: false, .. }) {
+                if !matches!(
+                    fn_abi.args[i].mode,
+                    PassMode::Indirect { mode: IndirectMode::Pointer, .. }
+                ) {
                     continue;
                 }
 
@@ -1353,10 +1356,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            let by_move = if let PassMode::Indirect { on_stack: false, .. } = fn_abi.args[i].mode
+            let by_move = if let PassMode::Indirect { mode: IndirectMode::Pointer, .. } =
+                fn_abi.args[i].mode
                 && kind == CallKind::Tail
             {
-                // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+                // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
                 //
                 // Normally an indirect argument that is allocated in the caller's stack frame
                 // would be passed as a pointer into the callee's stack frame.
@@ -1977,14 +1981,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
                 _ => bug!("codegen_argument: {:?} invalid for pair argument", op),
             },
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => match op.val {
-                Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => {
-                    llargs.push(a);
-                    llargs.push(b);
-                    return;
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
+                match op.val {
+                    Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => {
+                        llargs.push(a);
+                        llargs.push(b);
+                        return;
+                    }
+                    _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
                 }
-                _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
-            },
+            }
             _ => {}
         }
 
@@ -2014,7 +2020,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 PassMode::Ignore | PassMode::Pair(..) => unreachable!("handled above"),
             },
             Ref(op_place_val) => match arg.mode {
-                PassMode::Indirect { attrs, on_stack, .. } => {
+                PassMode::Indirect { attrs, mode, .. } => {
+                    if mode == IndirectMode::AmdgpuKernelArg {
+                        bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}");
+                    }
                     // For `foo(packed.large_field)`, and types with <4 byte alignment on x86,
                     // alignment requirements may be higher than the type's alignment, so copy
                     // to a higher-aligned alloca.
@@ -2023,7 +2032,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         None => arg.layout.align.abi,
                     };
                     // Copy to an alloca when the argument is neither by-val nor by-move.
-                    if op_place_val.align < required_align || (!on_stack && !by_move) {
+                    if op_place_val.align < required_align
+                        || (mode == IndirectMode::Pointer && !by_move)
+                    {
                         let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
                         bx.lifetime_start(scratch.llval, arg.layout.size);
                         op.store_with_annotation(bx, scratch.with_type(arg.layout));
@@ -2036,8 +2047,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 _ => (op_place_val.llval, op_place_val.align, true),
             },
             ZeroSized => match arg.mode {
-                PassMode::Indirect { on_stack, .. } => {
-                    if on_stack {
+                PassMode::Indirect { mode, .. } => {
+                    if mode == IndirectMode::AmdgpuKernelArg {
+                        bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}");
+                    }
+                    if mode == IndirectMode::OnStack {
                         // It doesn't seem like any target can have `byval` ZSTs, so this assert
                         // is here to replace a would-be untested codepath.
                         bug!("ZST {op:?} passed on stack with abi {arg:?}");
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index b5cecf4b5c434..aefa8356536dc 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -8,7 +8,7 @@ use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_span::{ErrorGuaranteed, bug, span_bug};
-use rustc_target::callconv::{FnAbi, PassMode};
+use rustc_target::callconv::{FnAbi, IndirectMode, PassMode};
 use tracing::{debug, instrument};
 
 use crate::base;
@@ -561,15 +561,21 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             match arg.mode {
                 // Sized indirect arguments
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+                PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
                     // Don't copy an indirect argument to an alloca, the caller already put it
                     // in a temporary alloca and gave it up.
+                    // AmdgpuKernelArg/byref arguments must not be modified, so always create a
+                    // local alloca for them.
+                    // If the argument is underaligned, then we need to copy it to a higher-aligned
+                    // alloca.
                     // FIXME: lifetimes
+                    let mut needs_alloca = mode == IndirectMode::AmdgpuKernelArg;
                     if let Some(pointee_align) = attrs.pointee_align
                         && pointee_align < arg.layout.align.abi
                     {
-                        // ...unless the argument is underaligned, then we need to copy it to
-                        // a higher-aligned alloca.
+                        needs_alloca = true;
+                    }
+                    if needs_alloca {
                         let tmp = PlaceRef::alloca(bx, arg.layout);
                         bx.store_fn_arg(arg, &mut llarg_idx, tmp);
                         LocalRef::Place(tmp)
@@ -580,7 +586,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                     }
                 }
                 // Unsized indirect arguments
-                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                     // As the storage for the indirect argument lives during
                     // the whole function call, we just copy the wide pointer.
                     let llarg = bx.get_param(llarg_idx);
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index 7f8edae4f17c3..68bb5e0a592bf 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -2856,7 +2856,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         true
     }
 
-    /// Identify some cases where `as_ref()` would be appropriate and suggest it.
+    /// Identify some cases where `as_ref()` or `as_mut()` would be appropriate and suggest it.
     ///
     /// Given the following code:
     /// ```compile_fail,E0308
@@ -2872,7 +2872,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// ```ignore (illustrative)
     /// opt.map(|param| { takes_ref(param) });
     /// ```
-    fn can_use_as_ref(&self, expr: &hir::Expr<'_>) -> Option<(Vec<(Span, String)>, &'static str)> {
+    fn can_use_as_ref_or_mut(
+        &self,
+        expr: &hir::Expr<'_>,
+        mutability: hir::Mutability,
+    ) -> Option<(Vec<(Span, String)>, &'static str)> {
         let hir::ExprKind::Path(hir::QPath::Resolved(_, path)) = expr.kind else {
             return None;
         };
@@ -2909,9 +2913,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return None;
         };
 
-        let self_ty = self.typeck_results.borrow().expr_ty_opt(receiver)?;
+        let mut self_ty = self.typeck_results.borrow().expr_ty_opt(receiver)?;
+        while let ty::Ref(_, inner, ref_mutability) = self_ty.kind() {
+            // `as_mut()` cannot borrow through a shared reference,
+            // also we cannot suggest `as_ref()` either when the reference is shared
+            if mutability.is_mut() && ref_mutability.is_not() {
+                return None;
+            }
+            self_ty = *inner;
+        }
         let name = method_path.ident.name;
-        let is_as_ref_able = match self_ty.peel_refs().kind() {
+        let can_borrow = match self_ty.kind() {
             ty::Adt(def, _) => {
                 (self.tcx.is_diagnostic_item(sym::Option, def.did())
                     || self.tcx.is_diagnostic_item(sym::Result, def.did()))
@@ -2919,11 +2931,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
             _ => false,
         };
-        if is_as_ref_able {
-            Some((
-                vec![(method_path.ident.span.shrink_to_lo(), "as_ref().".to_string())],
-                "consider using `as_ref` instead",
-            ))
+        if can_borrow {
+            let (suggestion, message) = match mutability {
+                hir::Mutability::Not => ("as_ref().", "consider using `as_ref` instead"),
+                hir::Mutability::Mut => ("as_mut().", "consider using `as_mut` instead"),
+            };
+            Some((vec![(method_path.ident.span.shrink_to_lo(), suggestion.to_string())], message))
         } else {
             None
         }
@@ -3114,7 +3127,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         return Some((suggs, help, app, mutref));
                     }
 
-                    if let Some((sugg, msg)) = self.can_use_as_ref(expr) {
+                    if let Some((sugg, msg)) = self.can_use_as_ref_or_mut(expr, mutability) {
                         return Some((
                             sugg,
                             msg.to_string(),
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index 161b5bdb952d3..bc8fa60b66a52 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -480,6 +480,11 @@ extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C,
   return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty)));
 }
 
+extern "C" LLVMAttributeRef LLVMRustCreateByRefAttr(LLVMContextRef C,
+                                                    LLVMTypeRef Ty) {
+  return wrap(Attribute::getWithByRefType(*unwrap(C), unwrap(Ty)));
+}
+
 extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C,
                                                         LLVMTypeRef Ty) {
   return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty)));
diff --git a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp
index bed54da0a7045..ca4d181179ffd 100644
--- a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp
@@ -28,7 +28,6 @@
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/Cloning.h"
-#include "llvm/Transforms/Utils/ModuleUtils.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 
 #include <optional>
@@ -88,22 +87,6 @@ extern "C" bool LLVMRustBundleImages(LLVMModuleRef M, TargetMachine &TM,
   return true;
 }
 
-extern "C" bool LLVMRustOffloadEmbedBufferInModule(LLVMModuleRef HostM,
-                                                   const char *HostOutPath) {
-  auto MBOrErr = MemoryBuffer::getFile(HostOutPath);
-  if (!MBOrErr) {
-    auto E = MBOrErr.getError();
-    auto _B = errorCodeToError(E);
-    return false;
-  }
-  MemoryBufferRef Buf = (*MBOrErr)->getMemBufferRef();
-  Module *M = unwrap(HostM);
-  StringRef SectionName = ".llvm.offloading";
-  Align Alignment = Align(8);
-  llvm::embedBufferInModule(*M, Buf, SectionName, Alignment);
-  return true;
-}
-
 // Clone OldFn into NewFn, remapping its arguments to RebuiltArgs.
 // Each arg of OldFn is replaced with the corresponding value in RebuiltArgs.
 // For scalars, RebuiltArgs contains the value cast and/or truncated to the
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index 5bba125aefc58..8814670ca4300 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -135,7 +135,7 @@ impl<'tcx> Visitor<'tcx> for DeduceParamAttrs {
             }
 
             // Like a call, but more conservative because the backend may introduce writes to an
-            // argument if the argument is passed as `PassMode::Indirect { on_stack: false, ... }`.
+            // argument if the argument is passed as `PassMode::Indirect { mode: IndirectMode::Pointer, ... }`.
             TerminatorKind::TailCall { .. } => {
                 for usage in self.usage.iter_mut() {
                     *usage |= UsageSummary::MUTATE;
diff --git a/compiler/rustc_public/src/abi.rs b/compiler/rustc_public/src/abi.rs
index 67d609c780c42..72387fce19797 100644
--- a/compiler/rustc_public/src/abi.rs
+++ b/compiler/rustc_public/src/abi.rs
@@ -41,6 +41,19 @@ pub struct ArgAbi {
     pub mode: PassMode,
 }
 
+/// Different modes in which indirect arguments can be passed.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize)]
+pub enum IndirectMode {
+    /// Passed as a normal pointer, nothing special.
+    Pointer,
+    /// The value is placed at a fixed stack offset rather than passed as a regular pointer
+    /// argument.
+    OnStack,
+    /// Similar to `OnStack` except that the pointer does not necessarily point to the stack, no
+    /// extra copy is made, and the passed argument should not be modified.
+    AmdgpuKernelArg,
+}
+
 /// How a function argument should be passed in to the target function.
 ///
 /// The pass mode is determined by the platform's calling convention and the
@@ -74,14 +87,13 @@ pub enum PassMode {
     /// Pass the argument indirectly via a pointer.
     ///
     /// The caller places the value in memory and passes a pointer to it.
-    /// When `on_stack` is true, the value is placed at a fixed stack offset
-    /// rather than passed as a regular pointer argument.
     Indirect {
         attrs: ArgAttributes,
         /// Attributes for the metadata pointer (vtable or length) of unsized arguments.
         /// Only present for unsized types (e.g., `dyn Trait`, `[T]`).
         meta_attrs: Option<ArgAttributes>,
-        on_stack: bool,
+        address_space: Option<AddressSpace>,
+        mode: IndirectMode,
     },
 }
 
diff --git a/compiler/rustc_public/src/unstable/convert/stable/abi.rs b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
index 3c268a6dd23a4..1ac0b706ed091 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/abi.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
@@ -11,9 +11,9 @@ use rustc_target::callconv;
 use crate::IndexedVal;
 use crate::abi::{
     AddressSpace, ArgAbi, ArgAttributes, ArgExtension, CallConvention, CastTarget, FieldsShape,
-    FloatLength, FnAbi, IntegerLength, IntegerType, Layout, LayoutShape, NumScalableVectors,
-    PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar, TagEncoding, TyAndLayout,
-    Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange,
+    FloatLength, FnAbi, IndirectMode, IntegerLength, IntegerType, Layout, LayoutShape,
+    NumScalableVectors, PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar,
+    TagEncoding, TyAndLayout, Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange,
 };
 use crate::compiler_interface::BridgeTys;
 use crate::target::MachineSize as Size;
@@ -155,6 +155,22 @@ impl<'tcx> Stable<'tcx> for CanonAbi {
     }
 }
 
+impl<'tcx> Stable<'tcx> for callconv::IndirectMode {
+    type T = IndirectMode;
+
+    fn stable<'cx>(
+        &self,
+        _tables: &mut Tables<'cx, BridgeTys>,
+        _cx: &CompilerCtxt<'cx, BridgeTys>,
+    ) -> Self::T {
+        match self {
+            callconv::IndirectMode::Pointer => IndirectMode::Pointer,
+            callconv::IndirectMode::OnStack => IndirectMode::OnStack,
+            callconv::IndirectMode::AmdgpuKernelArg => IndirectMode::AmdgpuKernelArg,
+        }
+    }
+}
+
 impl<'tcx> Stable<'tcx> for callconv::PassMode {
     type T = PassMode;
 
@@ -172,11 +188,14 @@ impl<'tcx> Stable<'tcx> for callconv::PassMode {
             callconv::PassMode::Cast { pad_i32_count, cast } => {
                 PassMode::Cast { pad_i32_count: *pad_i32_count, cast: cast.stable(tables, cx) }
             }
-            callconv::PassMode::Indirect { attrs, meta_attrs, on_stack } => PassMode::Indirect {
-                attrs: attrs.stable(tables, cx),
-                meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)),
-                on_stack: *on_stack,
-            },
+            callconv::PassMode::Indirect { attrs, meta_attrs, address_space, mode } => {
+                PassMode::Indirect {
+                    attrs: attrs.stable(tables, cx),
+                    meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)),
+                    address_space: address_space.stable(tables, cx),
+                    mode: mode.stable(tables, cx),
+                }
+            }
         }
     }
 }
diff --git a/compiler/rustc_target/src/callconv/amdgpu.rs b/compiler/rustc_target/src/callconv/amdgpu.rs
index 98ab3ce8eb746..7a9eeaba19c96 100644
--- a/compiler/rustc_target/src/callconv/amdgpu.rs
+++ b/compiler/rustc_target/src/callconv/amdgpu.rs
@@ -1,25 +1,60 @@
-use rustc_abi::{HasDataLayout, TyAbiInterface};
+use rustc_abi::{
+    AddressSpace, BackendRepr, CanonAbi, HasDataLayout, Reg, RegKind, TyAbiInterface, TyAndLayout,
+};
 
-use crate::callconv::{ArgAbi, FnAbi};
+use crate::callconv::{FnAbi, Uniform};
 
-fn classify_ret<'a, Ty, C>(_cx: &C, ret: &mut ArgAbi<'a, Ty>)
-where
-    Ty: TyAbiInterface<'a, C> + Copy,
-    C: HasDataLayout,
-{
-    ret.extend_integer_width_to(32);
-}
+// For reference, see llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp
 
-fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
+/// If the given type is a (potentially nested) struct containing a single scalar, return
+/// a `Uniform` for the contained, single element.
+fn single_element_struct_to_reg<'a, Ty, C>(cx: &C, ty: TyAndLayout<'a, Ty>) -> Option<Uniform>
 where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
-        arg.make_indirect();
-        return;
+    assert!(ty.is_aggregate(), "Only handles aggregate types");
+    if ty.layout.fields.count() != 1 {
+        return None;
+    }
+    let field = ty.field(cx, 0);
+    match field.backend_repr {
+        BackendRepr::SimdScalableVector { .. } => panic!("scalable vectors are unsupported"),
+        BackendRepr::Scalar(_) => {
+            // Check that the size is the same as the size for ty, so no extra padding
+            let size = field.layout.size.bytes();
+            if ty.layout.size.bytes() != size {
+                return None;
+            }
+
+            // clang passes the inner type directly, we emulate it with fitting integer types
+            match size {
+                1 => Some(Uniform::new(Reg::i8(), field.layout.size)),
+                2 => Some(Uniform::new(Reg::i16(), field.layout.size)),
+                4 => Some(Uniform::new(Reg::i32(), field.layout.size)),
+                8 => Some(Uniform::new(Reg::i64(), field.layout.size)),
+                16 => Some(Uniform::new(Reg::i128(), field.layout.size)),
+                s => panic!("Unhandled scalar of size {s} in amdgpu gpu-kernel ABI"),
+            }
+        }
+        BackendRepr::SimdVector { element, .. } => {
+            // Check that the size is the same as the size for ty, so no extra padding
+            let size = field.layout.size.bytes();
+            if ty.layout.size.bytes() != size {
+                return None;
+            }
+
+            // clang passes the inner type directly, we emulate it with a vector of the same type.
+            // The size is rounded up to the size of the complete type (including alignment).
+            let reg = Reg {
+                kind: RegKind::Vector { hint_vector_elem: element.primitive() },
+                size: field.layout.size,
+            };
+            Some(Uniform::new(reg, field.layout.size))
+        }
+        BackendRepr::Memory { .. } => single_element_struct_to_reg(cx, field),
+        BackendRepr::ScalarPair { .. } => None,
     }
-    arg.extend_integer_width_to(32);
 }
 
 pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
@@ -27,14 +62,25 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if !fn_abi.ret.is_ignore() {
-        classify_ret(cx, &mut fn_abi.ret);
-    }
+    // Kernels cannot return values, so do not handle return types
 
+    // Try to fill first registers with values and pass by_ref pointers for later indirect arguments
     for arg in fn_abi.args.iter_mut() {
         if arg.is_ignore() {
             continue;
         }
-        classify_arg(cx, arg);
+        if fn_abi.conv == CanonAbi::GpuKernel {
+            if arg.layout.is_aggregate() {
+                if let Some(uniform) = single_element_struct_to_reg(cx, arg.layout) {
+                    // Single element structs are passed directly as the inner type
+                    arg.cast_to(uniform);
+                } else {
+                    // All other aggregates are passed as by_ref pointer in the constant address space
+                    arg.pass_amdgpu_kernel_arg(Some(AddressSpace::GPU_CONSTANT));
+                }
+            }
+        } else {
+            // FIXME: C ABI is not yet implemented
+        }
     }
 }
diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs
index 1ac6168035085..edc23b6c50b45 100644
--- a/compiler/rustc_target/src/callconv/mod.rs
+++ b/compiler/rustc_target/src/callconv/mod.rs
@@ -36,6 +36,25 @@ mod x86_win32;
 mod x86_win64;
 mod xtensa;
 
+/// Different modes in which indirect arguments can be passed.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, StableHash)]
+pub enum IndirectMode {
+    /// Passed as a normal pointer, nothing special.
+    Pointer,
+    /// The value should be passed at a fixed stack offset in accordance to
+    /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
+    /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
+    /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
+    /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
+    /// alignment (if `None`). This means that the alignment will not always
+    /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
+    OnStack,
+    /// `AmdgpuKernelArg` behaves similar to `OnStack` except that the pointer does not necessarily
+    /// point to the stack, no extra copy is made, and the passed argument should not be modified.
+    /// This corresponds to the `byref` LLVM argument attribute.
+    AmdgpuKernelArg,
+}
+
 #[derive(Clone, PartialEq, Eq, Hash, Debug, StableHash)]
 pub enum PassMode {
     /// Ignore the argument.
@@ -63,16 +82,17 @@ pub enum PassMode {
     /// The `meta_attrs` value, if any, is for the metadata (vtable or length) of an unsized
     /// argument. (This is the only mode that supports unsized arguments.)
     ///
-    /// `on_stack` defines that the value should be passed at a fixed stack offset in accordance to
-    /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
-    /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
-    /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
-    /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
-    /// alignment (if `None`). This means that the alignment will not always
-    /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
+    /// `address_space` specifies if the pointer is in a special address space or the default one.
     ///
-    /// `on_stack` cannot be true for unsized arguments, i.e., when `meta_attrs` is `Some`.
-    Indirect { attrs: ArgAttributes, meta_attrs: Option<ArgAttributes>, on_stack: bool },
+    /// `mode` can be a special way to pass an argument indirectly.
+    /// `OnStack` and `AmdgpuKernelArg` cannot be used for unsized arguments, i.e., when
+    /// `meta_attrs` is `Some`.
+    Indirect {
+        attrs: ArgAttributes,
+        meta_attrs: Option<ArgAttributes>,
+        address_space: Option<AddressSpace>,
+        mode: IndirectMode,
+    },
 }
 
 impl PassMode {
@@ -89,13 +109,23 @@ impl PassMode {
                 PassMode::Cast { cast: c2, pad_i32_count: pad2 },
             ) => c1.eq_abi(c2) && pad1 == pad2,
             (
-                PassMode::Indirect { attrs: a1, meta_attrs: None, on_stack: s1 },
-                PassMode::Indirect { attrs: a2, meta_attrs: None, on_stack: s2 },
-            ) => a1.eq_abi(a2) && s1 == s2,
+                PassMode::Indirect { attrs: a1, meta_attrs: None, address_space: as1, mode: m1 },
+                PassMode::Indirect { attrs: a2, meta_attrs: None, address_space: as2, mode: m2 },
+            ) => a1.eq_abi(a2) && as1 == as2 && m1 == m2,
             (
-                PassMode::Indirect { attrs: a1, meta_attrs: Some(e1), on_stack: s1 },
-                PassMode::Indirect { attrs: a2, meta_attrs: Some(e2), on_stack: s2 },
-            ) => a1.eq_abi(a2) && e1.eq_abi(e2) && s1 == s2,
+                PassMode::Indirect {
+                    attrs: a1,
+                    meta_attrs: Some(e1),
+                    address_space: as1,
+                    mode: m1,
+                },
+                PassMode::Indirect {
+                    attrs: a2,
+                    meta_attrs: Some(e2),
+                    address_space: as2,
+                    mode: m2,
+                },
+            ) => a1.eq_abi(a2) && as1 == as2 && e1.eq_abi(e2) && m1 == m2,
             _ => false,
         }
     }
@@ -424,7 +454,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
 
         let meta_attrs = layout.is_unsized().then_some(ArgAttributes::new());
 
-        PassMode::Indirect { attrs, meta_attrs, on_stack: false }
+        PassMode::Indirect { attrs, meta_attrs, address_space: None, mode: IndirectMode::Pointer }
     }
 
     /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
@@ -435,13 +465,31 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
             PassMode::Direct(_) | PassMode::Pair(_, _) => {
                 self.mode = Self::indirect_pass_mode(&self.layout);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => {
+            PassMode::Indirect {
+                attrs: _,
+                meta_attrs: _,
+                address_space: _,
+                mode: IndirectMode::Pointer,
+            } => {
                 // already indirect
             }
             _ => panic!("Tried to make {:?} indirect", self.mode),
         }
     }
 
+    /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
+    /// This is valid for both sized and unsized arguments.
+    #[track_caller]
+    pub fn make_indirect_addrspace(&mut self, addrspace: AddressSpace) {
+        self.make_indirect();
+        match self.mode {
+            PassMode::Indirect { ref mut address_space, .. } => {
+                *address_space = Some(addrspace);
+            }
+            _ => unreachable!(),
+        }
+    }
+
     /// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass
     /// ZSTs indirectly.
     #[track_caller]
@@ -450,7 +498,12 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
             PassMode::Ignore => {
                 self.mode = Self::indirect_pass_mode(&self.layout);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => {
+            PassMode::Indirect {
+                attrs: _,
+                meta_attrs: _,
+                address_space: _,
+                mode: IndirectMode::Pointer,
+            } => {
                 // already indirect
             }
             _ => panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode),
@@ -477,8 +530,8 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout");
         self.make_indirect();
         match self.mode {
-            PassMode::Indirect { ref mut attrs, meta_attrs: _, ref mut on_stack } => {
-                *on_stack = true;
+            PassMode::Indirect { ref mut attrs, meta_attrs: _, address_space: _, ref mut mode } => {
+                *mode = IndirectMode::OnStack;
 
                 // Some platforms, like 32-bit x86, change the alignment of the type when passing
                 // `byval`. Account for that.
@@ -492,6 +545,22 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         }
     }
 
+    /// Pass this argument indirectly.
+    /// This corresponds to the `byref` LLVM argument attribute.
+    ///
+    /// `address_space` specifies the address space of the passed pointer.
+    pub fn pass_amdgpu_kernel_arg(&mut self, addrspace: Option<AddressSpace>) {
+        assert!(!self.layout.is_unsized(), "used amdgpu kernel arg ABI for unsized layout");
+        self.make_indirect();
+        match self.mode {
+            PassMode::Indirect { attrs: _, meta_attrs: _, ref mut address_space, ref mut mode } => {
+                *mode = IndirectMode::AmdgpuKernelArg;
+                *address_space = addrspace;
+            }
+            _ => unreachable!(),
+        }
+    }
+
     pub fn extend_integer_width_to(&mut self, bits: u64) {
         // Only integers have signedness
         if let BackendRepr::Scalar(scalar) = self.layout.backend_repr
@@ -545,11 +614,17 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
     }
 
     pub fn is_sized_indirect(&self) -> bool {
-        matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ })
+        matches!(
+            self.mode,
+            PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ }
+        )
     }
 
     pub fn is_unsized_indirect(&self) -> bool {
-        matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ })
+        matches!(
+            self.mode,
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ }
+        )
     }
 
     pub fn is_ignore(&self) -> bool {
@@ -851,7 +926,7 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                     // Compute `Aggregate` ABI.
 
                     let is_indirect_not_on_stack =
-                        matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
+                        matches!(arg.mode, PassMode::Indirect { mode: IndirectMode::Pointer, .. });
                     assert!(is_indirect_not_on_stack);
 
                     let size = arg.layout.size;
@@ -962,7 +1037,7 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(ArgAbi<'_, usize>, 56);
-    static_assert_size!(FnAbi<'_, usize>, 80);
+    static_assert_size!(ArgAbi<'_, usize>, 64);
+    static_assert_size!(FnAbi<'_, usize>, 88);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_target/src/callconv/x86.rs b/compiler/rustc_target/src/callconv/x86.rs
index a1c59d885b7fc..3476f41e3bc75 100644
--- a/compiler/rustc_target/src/callconv/x86.rs
+++ b/compiler/rustc_target/src/callconv/x86.rs
@@ -200,12 +200,13 @@ pub(crate) fn fill_inregs<'a, Ty, C>(
 
     for arg in fn_abi.args.iter_mut() {
         let attrs = match arg.mode {
-            PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+            PassMode::Ignore
+            | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
                 continue;
             }
             PassMode::Direct(ref mut attrs) => attrs,
             PassMode::Pair(..)
-            | PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ }
+            | PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ }
             | PassMode::Cast { .. } => {
                 unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
             }
diff --git a/compiler/rustc_target/src/callconv/xtensa.rs b/compiler/rustc_target/src/callconv/xtensa.rs
index 4dc9fad650636..49005adeb33c0 100644
--- a/compiler/rustc_target/src/callconv/xtensa.rs
+++ b/compiler/rustc_target/src/callconv/xtensa.rs
@@ -7,7 +7,7 @@
 
 use rustc_abi::{BackendRepr, HasDataLayout, Size, TyAbiInterface};
 
-use crate::callconv::{ArgAbi, FnAbi, Reg, Uniform};
+use crate::callconv::{ArgAbi, FnAbi, IndirectMode, Reg, Uniform};
 use crate::spec::HasTargetSpec;
 
 const NUM_ARG_GPRS: u64 = 6;
@@ -29,8 +29,8 @@ where
     classify_arg_ty(cx, arg, &mut arg_gprs_left, true);
     // Ret args cannot be passed via stack, we lower to indirect and let the backend handle the invisible reference
     match arg.mode {
-        super::PassMode::Indirect { attrs: _, meta_attrs: _, ref mut on_stack } => {
-            *on_stack = false;
+        super::PassMode::Indirect { attrs: _, meta_attrs: _, address_space: _, ref mut mode } => {
+            *mode = IndirectMode::Pointer;
         }
         _ => {}
     }
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index bcba5839926db..a9d8aa23b1ea5 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -12,7 +12,9 @@ use rustc_middle::ty::layout::{
 use rustc_middle::ty::{self, InstanceKind, ShimKind, Ty, TyCtxt, Unnormalized};
 use rustc_span::def_id::DefId;
 use rustc_span::{DUMMY_SP, bug};
-use rustc_target::callconv::{AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, PassMode};
+use rustc_target::callconv::{
+    AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, IndirectMode, PassMode,
+};
 use tracing::debug;
 
 pub(crate) fn provide(providers: &mut Providers) {
@@ -444,15 +446,15 @@ fn fn_abi_sanity_check<'tcx>(
                 // omitted entirely in the calling convention.
                 assert!(arg.is_ignore());
             }
-            if let PassMode::Indirect { on_stack, .. } = arg.mode
+            if let PassMode::Indirect { mode, .. } = arg.mode
                 && spec_abi != ExternAbi::RustTail
             {
-                assert!(!on_stack, "rustic abi {spec_abi:?} shouldn't use on_stack");
+                assert!(mode == IndirectMode::Pointer, "rust abi must use plain pointer mode");
             }
         } else if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
             assert_matches!(
                 arg.mode,
-                PassMode::Indirect { on_stack: false, .. },
+                PassMode::Indirect { mode: IndirectMode::Pointer, .. },
                 "the {spec_abi} ABI does not implement `#[rustc_pass_indirectly_in_non_rustic_abis]`"
             );
         }
@@ -506,9 +508,9 @@ fn fn_abi_sanity_check<'tcx>(
                 // Indirect returns are arguments from an ABI perspective.
                 fn_arg_attrs_sanity_check(attrs, false);
             }
-            PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, on_stack } => {
+            PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, address_space: _, mode } => {
                 // With metadata. Must be unsized and not on the stack.
-                assert!(arg.layout.is_unsized() && !on_stack);
+                assert!(arg.layout.is_unsized() && *mode == IndirectMode::Pointer);
                 // Also, must not be `extern` type.
                 let tail = tcx.struct_tail_for_codegen(arg.layout.ty, cx.typing_env);
                 if matches!(tail.kind(), ty::Foreign(..)) {
diff --git a/library/Cargo.lock b/library/Cargo.lock
index 3ddddbd5985b7..98fa2c5117411 100644
--- a/library/Cargo.lock
+++ b/library/Cargo.lock
@@ -426,9 +426,9 @@ dependencies = [
 
 [[package]]
 name = "wasip2"
-version = "2.0.0+wasi-0.2.12"
+version = "2.0.1+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96744b5e833bfd2d84c6faa7569693ce9e04a47c0a4d5519e6442f18af1a0fed"
+checksum = "89aafd4b69fb41a64cfd5d7f214cde92ab422ec4d9bcd55fcc816d3f5f49bbf3"
 dependencies = [
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
@@ -437,9 +437,9 @@ dependencies = [
 
 [[package]]
 name = "wasip3"
-version = "0.8.0+wasi-0.3.0"
+version = "0.9.0+wasi-0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42a999271e77c083825863fc85404197523b3f3cebe9b402aece658ebea90065"
+checksum = "f1d5749fdf69bb9400562eba50f8f25f5cd800ef4b74300038b6f99ae7409674"
 dependencies = [
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
@@ -456,9 +456,9 @@ version = "0.61.100"
 
 [[package]]
 name = "wit-bindgen"
-version = "0.61.1"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e473fd0095479f9689ac7d2a52c427cc96bb2b973ace50238dfcc1ab1cd52d93"
+checksum = "53cb4b5556c3a791e86838ea287782bdafa704d55b0e68b5b81a3a16b9ea5f4b"
 dependencies = [
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
diff --git a/library/core/src/num/imp/bignum.rs b/library/core/src/num/imp/bignum.rs
index 95046e96fc8f2..5c2a19f39dfce 100644
--- a/library/core/src/num/imp/bignum.rs
+++ b/library/core/src/num/imp/bignum.rs
@@ -386,7 +386,7 @@ macro_rules! define_bignum {
 /// the digit type for `Big32x40`
 pub type Digit32 = u32;
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 define_bignum!(Big32x40: type=Digit32, n=40);
 
 /// The digit type for `Big64x20`.
@@ -395,12 +395,12 @@ pub type Digit64 = u64;
 #[cfg(target_pointer_width = "64")]
 define_bignum!(Big64x20: type=Digit64, n=20);
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 pub type Big = Big32x40;
 #[cfg(target_pointer_width = "64")]
 pub type Big = Big64x20;
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 pub type Digit = Digit32;
 #[cfg(target_pointer_width = "64")]
 pub type Digit = Digit64;
diff --git a/library/core/src/num/imp/flt2dec/strategy/dragon.rs b/library/core/src/num/imp/flt2dec/strategy/dragon.rs
index 4ae0d4b4ab0ba..b63b93034d57f 100644
--- a/library/core/src/num/imp/flt2dec/strategy/dragon.rs
+++ b/library/core/src/num/imp/flt2dec/strategy/dragon.rs
@@ -16,22 +16,22 @@ static POW10: [Digit; 10] =
     [1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000];
 // precalculated arrays of `Digit`s for 5^(2^n).
 // FIXME(#162879): these tables have u64 and u32 versions, future versions may be generated by macro.
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO16: [Digit; 2] = [0x86f26fc1, 0x23];
 #[cfg(target_pointer_width = "64")]
 static POW5TO16: [Digit; 1] = [0x2386f26fc1];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO32: [Digit; 3] = [0x85acef81, 0x2d6d415b, 0x4ee];
 #[cfg(target_pointer_width = "64")]
 static POW5TO32: [Digit; 2] = [0x2d6d415b85acef81, 0x4ee];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO64: [Digit; 5] = [0xbf6a1f01, 0x6e38ed64, 0xdaa797ed, 0xe93ff9f4, 0x184f03];
 #[cfg(target_pointer_width = "64")]
 static POW5TO64: [Digit; 3] = [0x6e38ed64bf6a1f01, 0xe93ff9f4daa797ed, 0x184f03];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO128: [Digit; 10] = [
     0x2e953e01, 0x3df9909, 0xf1538fd, 0x2374e42f, 0xd3cff5ec, 0xc404dc08, 0xbccdb0da, 0xa6337f19,
     0xe91f2603, 0x24e,
@@ -40,7 +40,7 @@ static POW5TO128: [Digit; 10] = [
 static POW5TO128: [Digit; 5] =
     [0x3df99092e953e01, 0x2374e42f0f1538fd, 0xc404dc08d3cff5ec, 0xa6337f19bccdb0da, 0x24ee91f2603];
 
-#[cfg(target_pointer_width = "32")]
+#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))]
 static POW5TO256: [Digit; 19] = [
     0x982e7c01, 0xbed3875b, 0xd8d99f72, 0x12152f87, 0x6bde50c6, 0xcf4a6e70, 0xd595d80f, 0x26b2716e,
     0xadc666b0, 0x1d153624, 0x3c42d35a, 0x63ff540e, 0xcc5573c0, 0x65f9ef17, 0x55bc28f2, 0x80dcc7f7,
diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml
index de5c1feadb298..1d35b42a53947 100644
--- a/library/std/Cargo.toml
+++ b/library/std/Cargo.toml
@@ -84,12 +84,12 @@ wasip1 = { version = "1.0.0", features = [
 ], default-features = false }
 
 [target.'cfg(all(target_os = "wasi", target_env = "p2"))'.dependencies]
-wasip2 = { version = '2.0.0', features = [
+wasip2 = { version = '2.0.1', features = [
     'rustc-dep-of-std',
 ], default-features = false }
 
 [target.'cfg(all(target_os = "wasi", target_env = "p3"))'.dependencies]
-wasip3 = { version = '0.8.0', features = [
+wasip3 = { version = '0.9.0', features = [
     'rustc-dep-of-std',
 ], default-features = false }
 
diff --git a/library/std/src/sys/process/unix/unsupported/wait_status.rs b/library/std/src/sys/process/unix/unsupported/wait_status.rs
index ac54824875813..b77e7ae835656 100644
--- a/library/std/src/sys/process/unix/unsupported/wait_status.rs
+++ b/library/std/src/sys/process/unix/unsupported/wait_status.rs
@@ -1,4 +1,4 @@
-//! Emulated wait status for non-Unix #[cfg(unix) platforms
+//! Emulated wait status for non-Unix `#[cfg(unix)]` platforms
 //!
 //! Separate module to facilitate testing against a real Unix implementation.
 
diff --git a/tests/assembly-llvm/tail-call-indirect.rs b/tests/assembly-llvm/tail-call-indirect.rs
index 2bc1743a9bafd..918283966b405 100644
--- a/tests/assembly-llvm/tail-call-indirect.rs
+++ b/tests/assembly-llvm/tail-call-indirect.rs
@@ -10,10 +10,10 @@
 #![no_core]
 #![crate_type = "lib"]
 
-// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
 //
-// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
-// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to
+// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack
 // frame is overwritten by the callee's stack frame.
 //
 // The solution is to write the argument into the caller's argument place (stored somewhere further
diff --git a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
new file mode 100644
index 0000000000000..bc83f6510a6fe
--- /dev/null
+++ b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
@@ -0,0 +1,187 @@
+//@ add-minicore
+//@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 -Copt-level=3
+//@ needs-llvm-components: amdgpu
+#![feature(no_core, abi_gpu_kernel, repr_simd)]
+#![no_core]
+#![allow(improper_gpu_kernel_arg)]
+
+extern crate minicore;
+use minicore::num::Complex;
+
+// Tests from llvm-project/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
+
+#[repr(simd)]
+pub struct I8X2([i8; 2]);
+
+#[repr(simd)]
+pub struct I16X2([i16; 2]);
+
+#[repr(simd)]
+pub struct I16X3([i16; 3]);
+
+#[repr(simd)]
+pub struct I16X4([i16; 4]);
+
+#[repr(simd)]
+pub struct I32X3([i32; 3]);
+
+#[repr(simd)]
+pub struct I32X4([i32; 4]);
+
+#[repr(C)]
+pub struct SingleElementStructArg<T> {
+    i: T,
+}
+
+#[repr(C)]
+pub struct NestedSingleElementStructArg {
+    i: SingleElementStructArg<i32>,
+}
+
+#[repr(C)]
+pub struct StructArg {
+    i1: i32,
+    f: f32,
+    i2: i32,
+}
+
+#[repr(C)]
+pub struct StructPaddingArg {
+    i1: i8,
+    f: i64,
+}
+
+#[repr(C)]
+pub struct StructOfArraysArg {
+    i1: [i32; 2],
+    f1: f32,
+    i2: [i32; 4],
+    f2: [f32; 3],
+    i3: i32,
+}
+
+#[repr(C)]
+pub struct StructOfStructsArg {
+    i1: i32,
+    f1: f32,
+    s1: StructArg,
+    i2: i32,
+}
+
+#[repr(C)]
+pub union U {
+    b1: i32,
+    b2: f32,
+}
+
+#[repr(C)]
+pub struct SingleArrayElementStructArg {
+    i: [i32; 4],
+}
+
+#[repr(C)]
+pub struct SingleStructElementStructArgInner {
+    i: i32,
+    b: i64,
+}
+
+#[repr(C)]
+pub struct SingleStructElementStructArg {
+    s: SingleStructElementStructArgInner,
+}
+
+#[repr(C)]
+pub struct DifferentSizeTypePair {
+    l: i64,
+    i: i32,
+}
+
+// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(i32 %0)
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg<i32>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(i32 %0)
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_nested_single_element_struct_arg(
+    _: NestedSingleElementStructArg,
+) {
+}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([12 x i8]) align 4 captures(none) dereferenceable(12) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_arg(_: StructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_padding_arg(_: StructPaddingArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr addrspace(4) noalias nofree noundef readnone byref([44 x i8]) align 4 captures(none) dereferenceable(44) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_of_arrays_arg(_: StructOfArraysArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr addrspace(4) noalias nofree noundef readnone byref([24 x i8]) align 4 captures(none) dereferenceable(24) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_of_structs_arg(_: StructOfStructsArg) {}
+
+// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr addrspace(4) noalias nofree noundef readnone byref([4 x i8]) align 4 captures(none) dereferenceable(4) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn test_kernel_union_arg(_: U) {}
+
+// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 4 captures(none) dereferenceable(16) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_array_element_struct_arg(_: SingleArrayElementStructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_struct_element_struct_arg(
+    _: SingleStructElementStructArg,
+) {
+}
+
+// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_different_size_type_pair_arg(_: DifferentSizeTypePair) {}
+
+// CHECK: define amdgpu_kernel void @kernel_complex(ptr addrspace(4) noalias nofree noundef readnone byref([8 x i8]) align 4 captures(none) dereferenceable(8) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_complex(_: Complex<f32>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_slice(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_slice(_: &[u32]) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i64(i64 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i64(_: i64) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i64_struct(i64 {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i64_struct(_: SingleElementStructArg<i64>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i128_struct(i128 {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i128_struct(_: SingleElementStructArg<i128>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i8x2_struct(<2 x i8> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i8x2_struct(_: SingleElementStructArg<I8X2>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x2_struct(<2 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x2_struct(_: SingleElementStructArg<I16X2>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x3_struct(<4 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x3_struct(_: SingleElementStructArg<I16X3>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x4_struct(<4 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x4_struct(_: SingleElementStructArg<I16X4>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i32x3_struct(<4 x i32> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i32x3_struct(_: SingleElementStructArg<I32X3>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i32x4_struct(<4 x i32> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i32x4_struct(_: SingleElementStructArg<I32X4>) {}
diff --git a/tests/run-make/offload/host-std-device-nostd/rmake.rs b/tests/run-make/offload/host-std-device-nostd/rmake.rs
index 35f408e8b1cd0..6769993f9cf98 100644
--- a/tests/run-make/offload/host-std-device-nostd/rmake.rs
+++ b/tests/run-make/offload/host-std-device-nostd/rmake.rs
@@ -35,6 +35,4 @@ fn main() {
         .arg("-Clto=fat")
         .emit("obj")
         .run();
-
-    assert!(cwd().join("host.o").exists());
 }
diff --git a/tests/ui-fulldeps/rustc_public/check_abi.rs b/tests/ui-fulldeps/rustc_public/check_abi.rs
index ed616cc4d9bb0..92d6f28f8fd8c 100644
--- a/tests/ui-fulldeps/rustc_public/check_abi.rs
+++ b/tests/ui-fulldeps/rustc_public/check_abi.rs
@@ -15,8 +15,8 @@ extern crate rustc_middle;
 extern crate rustc_public;
 
 use rustc_public::abi::{
-    ArgAbi, ArgExtension, CallConvention, FieldsShape, IntegerLength, PassMode, Primitive, Scalar,
-    ValueRepr, VariantsShape,
+    ArgAbi, ArgExtension, CallConvention, FieldsShape, IndirectMode, IntegerLength, PassMode,
+    Primitive, Scalar, ValueRepr, VariantsShape,
 };
 use rustc_public::mir::MirVisitor;
 use rustc_public::mir::mono::Instance;
@@ -127,14 +127,14 @@ fn check_primitive(abi: &ArgAbi) {
 /// Check the return value: `Result<usize, &str>`.
 fn check_result(abi: &ArgAbi) {
     assert!(abi.ty.kind().is_enum());
-    let PassMode::Indirect { ref attrs, ref meta_attrs, on_stack } = abi.mode else {
+    let PassMode::Indirect { ref attrs, ref meta_attrs, address_space: _, mode } = abi.mode else {
         panic!("Expected PassMode::Indirect for Result, got: {:?}", abi.mode);
     };
     // Indirect arguments have a pointee alignment (the pointer must be aligned).
     assert!(attrs.pointee_align().is_some());
     // Result is a sized type, so no metadata pointer.
     assert!(meta_attrs.is_none());
-    assert!(!on_stack);
+    assert!(mode == IndirectMode::Pointer);
     let layout = abi.layout.shape();
     assert!(layout.is_sized());
     assert_matches!(layout.fields, FieldsShape::Arbitrary { .. });
diff --git a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
index 0bd4ac684066e..a54abdd5deeaf 100644
--- a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
+++ b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
@@ -23,7 +23,7 @@ use std::convert::TryFrom;
 use std::io::Write;
 use std::ops::ControlFlow;
 
-use rustc_public::abi::{CallConvention, PassMode, RegKind};
+use rustc_public::abi::{CallConvention, IndirectMode, PassMode, RegKind};
 use rustc_public::mir::mono::Instance;
 use rustc_public::{CrateDef, ItemKind};
 
@@ -147,7 +147,7 @@ fn test_abi_cast() -> ControlFlow<()> {
     }
     // Fourth TwoWords has no registers left → Indirect (on stack)
     assert!(
-        matches!(&abi.args[3].mode, PassMode::Indirect { on_stack: true, .. }),
+        matches!(&abi.args[3].mode, PassMode::Indirect { mode: IndirectMode::OnStack, .. }),
         "Expected arg 3 to be Indirect on stack, got: {:?}",
         abi.args[3].mode
     );
diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.powerpc-linux.stderr
+++ b/tests/ui/abi/c-zst.powerpc-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.s390x-linux.stderr
+++ b/tests/ui/abi/c-zst.s390x-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.sparc-linux.stderr b/tests/ui/abi/c-zst.sparc-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.sparc-linux.stderr
+++ b/tests/ui/abi/c-zst.sparc-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.sparc-none.stderr b/tests/ui/abi/c-zst.sparc-none.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.sparc-none.stderr
+++ b/tests/ui/abi/c-zst.sparc-none.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.sparc64-linux.stderr
+++ b/tests/ui/abi/c-zst.sparc64-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
index 302ffe1efc8b8..e5cad2199491b 100644
--- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
+++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr
index 6242d93b09534..1793674fa462a 100644
--- a/tests/ui/abi/debug.generic.stderr
+++ b/tests/ui/abi/debug.generic.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr
index 176c68ecd4c7b..29ec7846101f1 100644
--- a/tests/ui/abi/debug.loongarch64.stderr
+++ b/tests/ui/abi/debug.loongarch64.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr
index 176c68ecd4c7b..29ec7846101f1 100644
--- a/tests/ui/abi/debug.riscv64.stderr
+++ b/tests/ui/abi/debug.riscv64.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/pass-indirectly-attr.rs b/tests/ui/abi/pass-indirectly-attr.rs
index 54aafc716587c..bb90b8354ea91 100644
--- a/tests/ui/abi/pass-indirectly-attr.rs
+++ b/tests/ui/abi/pass-indirectly-attr.rs
@@ -20,7 +20,7 @@ pub struct Type(u8);
 pub extern "C" fn extern_c(_: Type) {}
 //~^ ERROR fn_abi_of(extern_c) = FnAbi {
 //~| ERROR mode: Indirect
-//~| ERROR on_stack: false,
+//~| ERROR mode: Pointer,
 //~| ERROR conv: C,
 
 #[rustc_abi(debug)]
diff --git a/tests/ui/abi/pass-indirectly-attr.stderr b/tests/ui/abi/pass-indirectly-attr.stderr
index efeec0d86982b..5821e6279bb85 100644
--- a/tests/ui/abi/pass-indirectly-attr.stderr
+++ b/tests/ui/abi/pass-indirectly-attr.stderr
@@ -48,7 +48,8 @@ error: fn_abi_of(extern_c) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.rs b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.rs
new file mode 100644
index 0000000000000..912cdef5b60bb
--- /dev/null
+++ b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.rs
@@ -0,0 +1,8 @@
+//@ dont-require-annotations: ERROR
+//@ compile-flags: --crate-type lib -Z ui-testing=no
+#![feature(rustc_attrs)]
+
+#[rustc_dyn_incompatible_trait]
+pub trait DynIncompatible {}
+
+pub fn f(_x: &dyn DynIncompatible) {}
diff --git a/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.stderr b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.stderr
new file mode 100644
index 0000000000000..c871061a57507
--- /dev/null
+++ b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait.stderr
@@ -0,0 +1,18 @@
+error[E0038]: the trait `DynIncompatible` is not dyn compatible
+ --> $DIR/rustc_dyn_incompatible_trait.rs:8:15
+  |
+8 | pub fn f(_x: &dyn DynIncompatible) {}
+  |               ^^^^^^^^^^^^^^^^^^^ `DynIncompatible` is not dyn compatible
+  |
+note: for a trait to be dyn compatible it needs to allow building a vtable
+      for more information, visit <https://doc.rust-lang.org/reference/items/traits.html#dyn-compatibility>
+ --> $DIR/rustc_dyn_incompatible_trait.rs:5:1
+  |
+5 | #[rustc_dyn_incompatible_trait]
+  | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...because it opted out of dyn-compatibility
+6 | pub trait DynIncompatible {}
+  |           --------------- this trait is not dyn compatible...
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0038`.
diff --git a/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs
new file mode 100644
index 0000000000000..fd8b3ec4e2761
--- /dev/null
+++ b/tests/ui/attributes/doc_examples/rustc_dyn_incompatible_trait2.rs
@@ -0,0 +1,52 @@
+//@run-fail
+mod foreign_crate {
+    /// # Safety requirements
+    ///
+    /// If this type also implements `SafeTrait`,
+    /// then that implementation must always return `true`.
+    pub unsafe trait UnsafeTrait {}
+    unsafe impl<T: UnsafeTrait + ?Sized> UnsafeTrait for &T {}
+
+    pub trait SafeTrait {
+        fn returns_true(&self) -> bool;
+    }
+    impl<T: SafeTrait + ?Sized> SafeTrait for &T {
+        fn returns_true(&self) -> bool {
+            (*self).returns_true()
+        }
+    }
+
+    impl SafeTrait for u8 {
+        fn returns_true(&self) -> bool {
+            true
+        }
+    }
+    /// Safety: impl returns `true`.
+    unsafe impl UnsafeTrait for u8 {}
+
+    pub fn function(x: impl UnsafeTrait + SafeTrait) {
+        // Can't panic, after all, `x: UnsafeTrait`
+        // guarantees `returns_true` actually returns `true`
+        assert!(x.returns_true());
+    }
+}
+
+use foreign_crate::{SafeTrait, UnsafeTrait, function};
+
+pub trait LocalTrait: UnsafeTrait {}
+impl<T: UnsafeTrait> LocalTrait for T {}
+
+// We can do this because `dyn LocalTrait` is a local type.
+// But `LocalTrait: UnsafeTrait`, so `dyn LocalTrait: UnsafeTrait` holds,
+// and we don't have to `unsafe impl` it.
+impl SafeTrait for dyn LocalTrait {
+    fn returns_true(&self) -> bool {
+        false
+    }
+}
+
+fn main() {
+    let x = 42_u8;
+    let y: &dyn LocalTrait = &x;
+    function(y); // panics
+}
diff --git a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
index 45edd7bc0e0ee..c9e77ac941901 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
+++ b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/c-variadic/pass-by-value-abi.rs b/tests/ui/c-variadic/pass-by-value-abi.rs
index bcca09e90438a..317840601c050 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.rs
+++ b/tests/ui/c-variadic/pass-by-value-abi.rs
@@ -27,9 +27,9 @@ use std::ffi::VaList;
 pub extern "C" fn take_va_list(_: VaList<'_>) {}
 //~^ ERROR fn_abi_of(take_va_list) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
 //[aarch64]~^^^^ ERROR mode: Indirect {
-//[aarch64]~^^^^^ ERROR on_stack: false,
+//[aarch64]~^^^^^ ERROR mode: Pointer,
 //[win]~^^^^^^ ERROR mode: Direct(
 
 #[cfg(all(target_arch = "x86_64", not(windows)))]
@@ -37,11 +37,11 @@ pub extern "C" fn take_va_list(_: VaList<'_>) {}
 pub extern "sysv64" fn take_va_list_sysv64(_: VaList<'_>) {}
 //[x86_64]~^ ERROR fn_abi_of(take_va_list_sysv64) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
 
 #[cfg(all(target_arch = "x86_64", not(windows)))]
 #[rustc_abi(debug)]
 pub extern "win64" fn take_va_list_win64(_: VaList<'_>) {}
 //[x86_64]~^ ERROR: fn_abi_of(take_va_list_win64) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
diff --git a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
index 1e203b93e66b3..04320a5312361 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
+++ b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -113,7 +114,8 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -193,7 +195,8 @@ error: fn_abi_of(take_va_list_win64) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/explicit-tail-calls/indirect.rs b/tests/ui/explicit-tail-calls/indirect.rs
index b3e2613efad25..71107ef420c35 100644
--- a/tests/ui/explicit-tail-calls/indirect.rs
+++ b/tests/ui/explicit-tail-calls/indirect.rs
@@ -25,17 +25,17 @@
 #![feature(explicit_tail_calls)]
 #![expect(incomplete_features)]
 
-// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
 //
-// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
-// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to
+// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack
 // frame is overwritten by the callee's stack frame.
 //
 // The solution is to write the argument into the caller's argument place (stored somewhere further
 // up the stack), and forward that place.
 
 // A struct big enough that it is not passed via registers, so that the rust calling convention uses
-// `Indirect { on_stack: false, .. }`.
+// `Indirect { mode: IndirectMode::Pointer, .. }`.
 #[repr(C)]
 #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Big([u64; 4]);
@@ -79,7 +79,7 @@ fn main() {
     assert_eq!(update_in_caller(Big::default()), 0 + 2 + 3 + 4);
 
     assert_eq!(swapper(u8::MIN, u8::MAX), (u8::MAX, u8::MIN));
-    // i128 uses `PassMode::Indirect { on_stack: false, .. }` on x86_64 MSVC.
+    // i128 uses `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` on x86_64 MSVC.
     assert_eq!(swapper(i128::MIN, i128::MAX), (i128::MAX, i128::MIN));
     assert_eq!(swapper(Big([1; 4]), Big([2; 4])), (Big([2; 4]), Big([1; 4])));
 
diff --git a/tests/ui/suggestions/as-mut-closure.fixed b/tests/ui/suggestions/as-mut-closure.fixed
new file mode 100644
index 0000000000000..76a9cc5ab7c88
--- /dev/null
+++ b/tests/ui/suggestions/as-mut-closure.fixed
@@ -0,0 +1,34 @@
+// Borrowing a container for a closure argument must preserve the expected mutability.
+//@ run-rustfix
+
+#![allow(unused_mut)]
+
+fn append(value: &mut String) {
+    value.push('!');
+}
+
+fn increment(value: &mut i32) {
+    *value += 1;
+}
+
+fn main() {
+    let mut option = Some(String::new());
+    let _ = option.as_mut().map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut option).as_mut().and_then(|arg| Some(append(arg)));
+    //~^ ERROR mismatched types
+
+    let mut result: Result<_, ()> = Ok(String::new());
+    let _ = result.as_mut().map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut result).as_mut().and_then(|arg| Ok(append(arg)));
+    //~^ ERROR mismatched types
+
+    // A shared reference cannot supply `as_mut()`. Borrow the copied argument instead.
+    let shared = &Some(0);
+    let _ = shared.map(|mut arg| increment(&mut arg));
+    //~^ ERROR mismatched types
+    let nested = &mut &Some(0);
+    let _ = nested.map(|mut arg| increment(&mut arg));
+    //~^ ERROR mismatched types
+}
diff --git a/tests/ui/suggestions/as-mut-closure.rs b/tests/ui/suggestions/as-mut-closure.rs
new file mode 100644
index 0000000000000..4b583cbc4f2dc
--- /dev/null
+++ b/tests/ui/suggestions/as-mut-closure.rs
@@ -0,0 +1,34 @@
+// Borrowing a container for a closure argument must preserve the expected mutability.
+//@ run-rustfix
+
+#![allow(unused_mut)]
+
+fn append(value: &mut String) {
+    value.push('!');
+}
+
+fn increment(value: &mut i32) {
+    *value += 1;
+}
+
+fn main() {
+    let mut option = Some(String::new());
+    let _ = option.map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut option).and_then(|arg| Some(append(arg)));
+    //~^ ERROR mismatched types
+
+    let mut result: Result<_, ()> = Ok(String::new());
+    let _ = result.map(|arg| append(arg));
+    //~^ ERROR mismatched types
+    let _ = (&mut result).and_then(|arg| Ok(append(arg)));
+    //~^ ERROR mismatched types
+
+    // A shared reference cannot supply `as_mut()`. Borrow the copied argument instead.
+    let shared = &Some(0);
+    let _ = shared.map(|mut arg| increment(arg));
+    //~^ ERROR mismatched types
+    let nested = &mut &Some(0);
+    let _ = nested.map(|mut arg| increment(arg));
+    //~^ ERROR mismatched types
+}
diff --git a/tests/ui/suggestions/as-mut-closure.stderr b/tests/ui/suggestions/as-mut-closure.stderr
new file mode 100644
index 0000000000000..8a998be539458
--- /dev/null
+++ b/tests/ui/suggestions/as-mut-closure.stderr
@@ -0,0 +1,111 @@
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:16:37
+   |
+LL |     let _ = option.map(|arg| append(arg));
+   |                              ------ ^^^ expected `&mut String`, found `String`
+   |                              |
+   |                              arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:6:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_mut` instead
+   |
+LL |     let _ = option.as_mut().map(|arg| append(arg));
+   |                    +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:18:54
+   |
+LL |     let _ = (&mut option).and_then(|arg| Some(append(arg)));
+   |                                               ------ ^^^ expected `&mut String`, found `String`
+   |                                               |
+   |                                               arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:6:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_mut` instead
+   |
+LL |     let _ = (&mut option).as_mut().and_then(|arg| Some(append(arg)));
+   |                           +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:22:37
+   |
+LL |     let _ = result.map(|arg| append(arg));
+   |                              ------ ^^^ expected `&mut String`, found `String`
+   |                              |
+   |                              arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:6:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_mut` instead
+   |
+LL |     let _ = result.as_mut().map(|arg| append(arg));
+   |                    +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:24:52
+   |
+LL |     let _ = (&mut result).and_then(|arg| Ok(append(arg)));
+   |                                             ------ ^^^ expected `&mut String`, found `String`
+   |                                             |
+   |                                             arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:6:4
+   |
+LL | fn append(value: &mut String) {
+   |    ^^^^^^ ------------------
+help: consider using `as_mut` instead
+   |
+LL |     let _ = (&mut result).as_mut().and_then(|arg| Ok(append(arg)));
+   |                           +++++++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:29:44
+   |
+LL |     let _ = shared.map(|mut arg| increment(arg));
+   |                                  --------- ^^^ expected `&mut i32`, found integer
+   |                                  |
+   |                                  arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:10:4
+   |
+LL | fn increment(value: &mut i32) {
+   |    ^^^^^^^^^ ---------------
+help: consider mutably borrowing here
+   |
+LL |     let _ = shared.map(|mut arg| increment(&mut arg));
+   |                                            ++++
+
+error[E0308]: mismatched types
+  --> $DIR/as-mut-closure.rs:32:44
+   |
+LL |     let _ = nested.map(|mut arg| increment(arg));
+   |                                  --------- ^^^ expected `&mut i32`, found integer
+   |                                  |
+   |                                  arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/as-mut-closure.rs:10:4
+   |
+LL | fn increment(value: &mut i32) {
+   |    ^^^^^^^^^ ---------------
+help: consider mutably borrowing here
+   |
+LL |     let _ = nested.map(|mut arg| increment(&mut arg));
+   |                                            ++++
+
+error: aborting due to 6 previous errors
+
+For more information about this error, try `rustc --explain E0308`.