diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 8b770528a1736..b3df1cbe4c5ab 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -246,7 +246,7 @@ use self::Ordering::*; use crate::cell::UnsafeCell; use crate::hint::spin_loop; -use crate::intrinsics::AtomicOrdering as AO; +use crate::intrinsics::{AtomicOrdering as AO, transmute_unchecked}; use crate::mem::transmute; use crate::{fmt, intrinsics}; @@ -276,14 +276,20 @@ mod private { /// A marker trait for primitive types which can be modified atomically. /// -/// This is an implementation detail for [Atomic]\ which may disappear or be replaced at any time. -// -// # Safety -// -// Types implementing this trait must be primitives that can be modified atomically. -// -// The associated `Self::Storage` type must have the same size, but may have fewer validity -// invariants or a higher alignment requirement than `Self`. +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives that can be modified atomically. +/// +/// The associated `Self::Storage` type must have the same size, but may have a higher alignment +/// requirement than `Self`. Transmuting `Self::Storage` to/from `Self` via `unchecked_transmute` +/// must be valid. This also implies that transmuting/casting a `Self::Storage` reference/pointer to +/// `Self` is allowed, however there is no requirement when it comes to transmuting/casting a +/// `Self` reference/pointer to `Self::Storage`. Transmuting a `Self` reference/pointer to +/// `Self::Storage` can only be done if the reference/pointer has the same alignment as +/// `Self::Storage`. #[unstable( feature = "atomic_internals", reason = "implementation detail which may disappear or be replaced at any time", @@ -294,76 +300,330 @@ pub impl(self) unsafe trait AtomicPrimitive: Sized + Copy { type Storage: Sized; } -macro impl_atomic_primitive { +/// A marker trait for primitive types which can perform load/store operations atomically. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives that can perform load/store operations +/// atomically. +/// +/// +/// The associated `Self::OpType` type must be an integer or pointer type that is the same size as +/// `Self`. This is the type used with the atomic_load and atomic_store functions. Transmuting +/// between `Self::OpType` and `Self` must be valid. Casting (and then dereferencing) a pointer +/// from one type to the other must also be valid. +/// +/// Types implementing this trait will have the following methods implemented for +/// [Atomic]\ automatically: +/// +/// - new +/// - from_ptr +/// - get_mut +/// - get_mut_slice +/// - into_inner +/// - load +/// - store +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicLoadStore: + Sized + Copy + AtomicPrimitive +{ + /// Temporary implementation detail. + type OpType: Sized + Copy; +} + +/// A market trait for primitive types which can be modified atomically, and don't have special +/// atomic emulation fallbacks. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives that can modified atomically, and have no +/// special handling or emulation fallback. Implementing this trait automatically implements the +/// following atomic methods for [Atomic]\: +/// +/// - swap +/// - compare_exchange +/// - compare_exchange_weak +/// - compare_and_swap +/// - fetch_update +/// - try_update +/// - update +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicCas: AtomicLoadStore {} + +/// A market trait for primitive types which have the same alignment as their atomic counter-parts. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives whose alignment is the same as their atomic +/// counter parts. Implementing this trait automatically implements the following methods for +/// [Atomic]\: +/// +/// - from_mut +/// - from_mut_slice +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicAlignedPrimitive: AtomicLoadStore {} + +/// A marker trait for atomic integer types. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitive integer types. Implementing this trait +/// automatically implements the following atomic methods for [Atomic]\: +/// +/// - fetch_add +/// - fetch_sub +/// - fetch_max +/// - fetch_min +/// +/// The associated constant Self::IS_SIGNED must correspond to whether the actual integer type this +/// trait is implemented for is signed (true) or not (false). +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicInteger: AtomicCas { + /// Whether the integer type is signed or not + const IS_SIGNED: bool; +} + +/// A marker trait for atomic types that support bitwise operations +/// +/// # Safety +/// +/// Types implementing this trait must be primitives which support atomic bitwise operations. +/// Implementing this trait automatically implements the following atomic methods for +/// [Atomic]\: +/// +/// - fetch_nand +/// - fetch_and +/// - fetch_or +/// - fetch xor +/// +/// # Note +/// +/// The bitwise and, or, and xor operation are defined for both atomic pointers and integers, but +/// are only automatically implemented for integers. This is because the existing documentation +/// for these operations on pointers would be trampled, and it currently provides very useful +/// information on their use in relation to the strict provenance API. +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicBitwise: AtomicCas {} + +macro_rules! impl_atomic_traits { ( @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, - $cfg:meta + size($size:literal), + load_store ) => { #[unstable( feature = "atomic_internals", reason = "implementation detail which may disappear or be replaced at any time", issue = "none" )] - #[cfg($cfg)] - unsafe impl $(<$T>)? AtomicPrimitive for $Primitive { - type Storage = private::$Storage<$Operand>; + #[cfg(any(target_has_atomic_load_store = $size, doc))] + unsafe impl $(<$T>)? AtomicLoadStore for $Primitive { + type OpType = $Operand; + } + }; + + ( + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + cas + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicCas for $Primitive {} + }; + + ( + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + aligned + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic_primitive_alignment = $size, doc))] + unsafe impl $(<$T>)? AtomicAlignedPrimitive for $Primitive {} + }; + + ( + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + signed + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicInteger for $Primitive { + const IS_SIGNED: bool = true; } - }, + }; ( - [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, - size($size:literal) + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + unsigned ) => { - impl_atomic_primitive!( - @impl [$($T)?] $Primitive as $Storage<$Operand>, - target_has_atomic_load_store = $size - ); - }, + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicInteger for $Primitive { + const IS_SIGNED: bool = false; + } + }; ( - [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, size($size:literal), - doc + bitwise + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicBitwise for $Primitive {} + }; + + ( + [$T:ident] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal) $(, $trait:ident)* + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic_load_store = $size, doc))] + unsafe impl <$T> AtomicPrimitive for $Primitive { + type Storage = private::$Storage<$Operand>; + } + + $( + impl_atomic_traits!(@impl [$T] $Primitive as $Storage<$Operand>, size($size), $trait); + )* + }; + + ( + $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal) $(, $trait:ident)* ) => { - impl_atomic_primitive!( - @impl [$($T)?] $Primitive as $Storage<$Operand>, - any(target_has_atomic_load_store = $size, doc) - ); - }, + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic_load_store = $size, doc))] + unsafe impl AtomicPrimitive for $Primitive { + type Storage = private::$Storage<$Operand>; + } + + $( + impl_atomic_traits!(@impl [] $Primitive as $Storage<$Operand>, size($size), $trait); + )* + }; } -impl_atomic_primitive!([] bool as Align1, size("8")); -impl_atomic_primitive!([] i8 as Align1, size("8")); -impl_atomic_primitive!([] u8 as Align1, size("8")); -impl_atomic_primitive!([] i16 as Align2, size("16")); -impl_atomic_primitive!([] u16 as Align2, size("16")); -impl_atomic_primitive!([] i32 as Align4, size("32")); -impl_atomic_primitive!([] u32 as Align4, size("32")); -impl_atomic_primitive!([] i64 as Align8, size("64")); -impl_atomic_primitive!([] u64 as Align8, size("64")); -impl_atomic_primitive!([] i128 as Align16, size("128"), doc); -impl_atomic_primitive!([] u128 as Align16, size("128"), doc); +impl_atomic_traits!(bool as Align1, size("8"), load_store, aligned); + +impl_atomic_traits!(i8 as Align1, size("8"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u8 as Align1, size("8"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(i16 as Align2, size("16"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u16 as Align2, size("16"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(i32 as Align4, size("32"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u32 as Align4, size("32"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(i64 as Align8, size("64"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u64 as Align8, size("64"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(u128 as Align16, size("128")); +impl_atomic_traits!(i128 as Align16, size("128")); #[cfg(target_pointer_width = "16")] -impl_atomic_primitive!([] isize as Align2, size("ptr")); +impl_atomic_traits!(isize as Align2, size("ptr"), load_store, cas, aligned, signed, bitwise); #[cfg(target_pointer_width = "32")] -impl_atomic_primitive!([] isize as Align4, size("ptr")); +impl_atomic_traits!(isize as Align4, size("ptr"), load_store, cas, aligned, signed, bitwise); #[cfg(target_pointer_width = "64")] -impl_atomic_primitive!([] isize as Align8, size("ptr")); +impl_atomic_traits!(isize as Align8, size("ptr"), load_store, cas, aligned, signed, bitwise); #[cfg(target_pointer_width = "16")] -impl_atomic_primitive!([] usize as Align2, size("ptr")); +impl_atomic_traits!( + usize as Align2, + size("ptr"), + load_store, + cas, + aligned, + unsigned, + bitwise +); #[cfg(target_pointer_width = "32")] -impl_atomic_primitive!([] usize as Align4, size("ptr")); +impl_atomic_traits!( + usize as Align4, + size("ptr"), + load_store, + cas, + aligned, + unsigned, + bitwise +); #[cfg(target_pointer_width = "64")] -impl_atomic_primitive!([] usize as Align8, size("ptr")); +impl_atomic_traits!( + usize as Align8, + size("ptr"), + load_store, + cas, + aligned, + unsigned, + bitwise +); #[cfg(target_pointer_width = "16")] -impl_atomic_primitive!([T] *mut T as Align2<*mut T>, size("ptr")); +impl_atomic_traits!([T] *mut T as Align2<*mut T>, size("ptr"), load_store, cas, aligned); #[cfg(target_pointer_width = "32")] -impl_atomic_primitive!([T] *mut T as Align4<*mut T>, size("ptr")); +impl_atomic_traits!([T] *mut T as Align4<*mut T>, size("ptr"), load_store, cas, aligned); #[cfg(target_pointer_width = "64")] -impl_atomic_primitive!([T] *mut T as Align8<*mut T>, size("ptr")); +impl_atomic_traits!([T] *mut T as Align8<*mut T>, size("ptr"), load_store, cas, aligned); /// A memory location which can be safely modified from multiple threads. /// @@ -416,16 +676,6 @@ const EMULATE_ATOMIC_BOOL: bool = cfg!(any( #[stable(feature = "rust1", since = "1.0.0")] pub type AtomicBool = Atomic; -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "rust1", since = "1.0.0")] -impl Default for AtomicBool { - /// Creates an `AtomicBool` initialized to `false`. - #[inline] - fn default() -> Self { - Self::new(false) - } -} - /// A raw pointer type which can be safely shared between threads. /// /// This type has the same size and bit validity as a `*mut T`. @@ -436,15 +686,6 @@ impl Default for AtomicBool { #[stable(feature = "rust1", since = "1.0.0")] pub type AtomicPtr = Atomic<*mut T>; -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "rust1", since = "1.0.0")] -impl Default for AtomicPtr { - /// Creates a null `AtomicPtr`. - fn default() -> AtomicPtr { - AtomicPtr::new(crate::ptr::null_mut()) - } -} - /// Atomic memory orderings /// /// Memory orderings specify the way atomic operations synchronize memory. @@ -526,45 +767,97 @@ pub enum Ordering { SeqCst, } -/// An [`AtomicBool`] initialized to `false`. -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "rust1", since = "1.0.0")] -#[deprecated( - since = "1.34.0", - note = "the `new` function is now preferred", - suggestion = "AtomicBool::new(false)" -)] -#[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] -pub const ATOMIC_BOOL_INIT: AtomicBool = AtomicBool::new(false); +impl Atomic { + /// Creates a new `Atomic` with the given value. + /// + /// # Examples + /// + /// ``` + /// use std::sync::atomic::AtomicBool; + /// + /// let atomic = AtomicBool::new(true); + /// ``` + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_stable(feature = "const_atomic_new", since = "1.34.0")] + #[must_use] + pub const fn new(v: T) -> Self { + Self { + // SAFETY: + // By the contract of AtomicPrimitive, it is guaranteed that transmuting between T and + // T::Storage is valid. + v: UnsafeCell::new(unsafe { transmute_unchecked(v) }), + } + } -#[cfg(target_has_atomic_load_store = "8")] -impl AtomicBool { - /// Creates a new `AtomicBool`. + /// Consumes the atomic and returns the contained value. /// /// # Examples /// /// ``` /// use std::sync::atomic::AtomicBool; /// - /// let atomic_true = AtomicBool::new(true); - /// let atomic_false = AtomicBool::new(false); + /// let some_bool = AtomicBool::new(true); + /// assert_eq!(some_bool.into_inner(), true); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_atomic_new", since = "1.24.0")] - #[must_use] - pub const fn new(v: bool) -> AtomicBool { + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0")] + pub const fn into_inner(self) -> T { // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(v) } + // * Consuming self means there are no references to self/the underlying data + // * The UnsafeCell can be transmuted into T::Storage which by the contract of + // AtomicPrimitive can in turn by transmuted into T + // * No operations on Atomic can produce a T::Storage that is an invalid T. + unsafe { + // We can't use UnsafeCell's into_inner despite it being "older" (present since 1.0.0) + // because it's const variant wasn't stabilized until 1.83.0 and the const variant of + // this function was stabilized earlier in 1.79.0 + transmute_unchecked(self) + } + } + + /// Returns a mutable pointer to the underlying `T`. + /// + /// Doing non-atomic reads and writes on the resulting value can be a data race. This is mostly + /// useful for FFI, where the function signature may use `*mut T` instead of `&Atomic`. + /// + /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the + /// atomic types work with interior mutability. All modifications of an atomic change the value + /// through a shared reference, and can do so safely as long as they use atomic operations. Any + /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the + /// requirements of the [memory model]. + /// + /// # Examples + /// + /// ```ignore (extern-declaration) + /// use std::sync::atomic::AtomicBool; + /// + /// extern "C" { + /// fn my_atomic_op(arg: *mut bool); + /// } + /// + /// let mut atomic = AtomicBool::new(true); + /// unsafe { + /// my_atomic_op(atomic.as_ptr()); + /// } + /// ``` + /// + /// [memory model]: self#memory-model-for-atomic-accesses + #[inline] + #[stable(feature = "atomic_as_ptr", since = "1.70.0")] + #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] + #[rustc_never_returns_null_ptr] + #[rustc_should_not_be_called_on_const_items] + pub const fn as_ptr(&self) -> *mut T { + self.v.get().cast() } - /// Creates a new `AtomicBool` from a pointer. + /// Creates a new `Atomic` from a pointer. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{self, AtomicBool}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// /// // Get a pointer to an allocated value /// let ptr: *mut bool = Box::into_raw(Box::new(false)); @@ -576,7 +869,7 @@ impl AtomicBool { /// let atomic = unsafe { AtomicBool::from_ptr(ptr) }; /// /// // Use `atomic` for atomic operations, possibly share it with other threads - /// atomic.store(true, atomic::Ordering::Relaxed); + /// atomic.store(true, Ordering::Relaxed); /// } /// /// // It's ok to non-atomically access the value behind `ptr`, @@ -589,8 +882,7 @@ impl AtomicBool { /// /// # Safety /// - /// * `ptr` must be aligned to `align_of::()` (note that this is always true, since - /// `align_of::() == 1`). + /// * `ptr` must be aligned to `align_of::>()`. /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`. /// * You must adhere to the [Memory model for atomic accesses]. In particular, it is not /// allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different @@ -601,25 +893,25 @@ impl AtomicBool { #[inline] #[stable(feature = "atomic_from_ptr", since = "1.75.0")] #[rustc_const_stable(feature = "const_atomic_from_ptr", since = "1.84.0")] - pub const unsafe fn from_ptr<'a>(ptr: *mut bool) -> &'a AtomicBool { + pub const unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Atomic { // SAFETY: guaranteed by the caller unsafe { &*ptr.cast() } } - /// Creates a new pointer to `AtomicBool` from a pointer. + /// Creates a new pointer to an atomic from a pointer. /// /// This is useful if you want to do volatile atomic accesses, and thus avoid creating /// a reference to the destination. #[inline] #[unstable(feature = "atomic_volatile", issue = "158947")] - pub const fn from_ptr_raw(ptr: *mut bool) -> *const AtomicBool { + pub const fn from_ptr_raw(ptr: *mut T) -> *const Self { ptr.cast_const().cast() } - /// Returns a mutable reference to the underlying [`bool`]. + /// Returns a mutable reference to the underlying `T`. /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. + /// This is safe because the mutable reference guarantees that no other threads are concurrently + /// accessing the atomic data. /// /// # Examples /// @@ -632,43 +924,24 @@ impl AtomicBool { /// assert_eq!(some_bool.load(Ordering::SeqCst), false); /// ``` #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut(&mut self) -> &mut bool { - // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *self.as_ptr() } - } - - /// Gets atomic access to a `&mut bool`. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let mut some_bool = true; - /// let a = AtomicBool::from_mut(&mut some_bool); - /// a.store(false, Ordering::Relaxed); - /// assert_eq!(some_bool, false); - /// ``` - #[inline] - #[cfg(target_has_atomic_primitive_alignment = "8")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut(v: &mut bool) -> &mut Self { - // SAFETY: the mutable reference guarantees unique ownership, and - // alignment of both `bool` and `Self` is 1. - unsafe { &mut *(v as *mut bool as *mut Self) } + pub const fn get_mut(&mut self) -> &mut T { + // SAFETY: + // * The mutable reference guarantees unique ownership. + // * The contract of AtomicPrimitive guarantees that transmuting &mut T::Storage to &mut T + // is valid. + unsafe { transmute(self.v.get_mut()) } } - /// Gets non-atomic access to a `&mut [AtomicBool]` slice. + /// Gets non-atomic access to a `&mut [Atomic]` slice. /// /// This is safe because the mutable reference guarantees that no other threads are /// concurrently accessing the atomic data. /// /// # Examples /// - /// ```ignore-wasm + /// ```rust,ignore-wasm /// use std::sync::atomic::{AtomicBool, Ordering}; /// /// let mut some_bools = [const { AtomicBool::new(false) }; 10]; @@ -690,73 +963,23 @@ impl AtomicBool { #[inline] #[stable(feature = "atomic_from_mut", since = "1.98.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut_slice(this: &mut [Self]) -> &mut [bool] { + pub const fn get_mut_slice(this: &mut [Self]) -> &mut [T] { // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *(this as *mut [Self] as *mut [bool]) } + unsafe { &mut *(this as *mut [Self] as *mut [T]) } } - /// Gets atomic access to a `&mut [bool]` slice. + /// Loads a value atomically. + /// + /// `load` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// + /// # Panics + /// + /// Panics if `order` is [`Release`] or [`AcqRel`]. /// /// # Examples /// - /// ```rust,ignore-wasm - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let mut some_bools = [false; 10]; - /// let a = &*AtomicBool::from_mut_slice(&mut some_bools); - /// std::thread::scope(|s| { - /// for i in 0..a.len() { - /// s.spawn(move || a[i].store(true, Ordering::Relaxed)); - /// } - /// }); - /// assert_eq!(some_bools, [true; 10]); - /// ``` - #[inline] - #[cfg(target_has_atomic_primitive_alignment = "8")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut_slice(v: &mut [bool]) -> &mut [Self] { - // SAFETY: the mutable reference guarantees unique ownership, and - // alignment of both `bool` and `Self` is 1. - unsafe { &mut *(v as *mut [bool] as *mut [Self]) } - } - - /// Consumes the atomic and returns the contained value. - /// - /// This is safe because passing `self` by value guarantees that no other threads are - /// concurrently accessing the atomic data. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::AtomicBool; - /// - /// let some_bool = AtomicBool::new(true); - /// assert_eq!(some_bool.into_inner(), true); - /// ``` - #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0")] - pub const fn into_inner(self) -> bool { - // SAFETY: - // * `Atomic` is essentially a transparent wrapper around `T`. - // * all operations on `Atomic` ensure that `T::Storage` remains - // a valid `bool`. - unsafe { transmute(self) } - } - - /// Loads a value from the bool. - /// - /// `load` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. - /// - /// # Panics - /// - /// Panics if `order` is [`Release`] or [`AcqRel`]. - /// - /// # Examples - /// - /// ``` + /// ``` /// use std::sync::atomic::{AtomicBool, Ordering}; /// /// let some_bool = AtomicBool::new(true); @@ -764,18 +987,21 @@ impl AtomicBool { /// assert_eq!(some_bool.load(Ordering::Relaxed), true); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const fn load(&self, order: Ordering) -> bool { + pub const fn load(&self, order: Ordering) -> T { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. unsafe { - atomic_load::<_, /* VOLATILE */ false>(self.v.get().cast::(), order) != 0 + transmute_unchecked(atomic_load::<_, /* VOLATILE */ false>( + self.v.get().cast::(), + order, + )) } } - /// Perform a volatile atomic load from the bool. + /// Perform a volatile atomic load. /// /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. @@ -798,18 +1024,48 @@ impl AtomicBool { /// # Panics /// /// Panics if `order` is [`Release`] or [`AcqRel`]. + /// + /// # Examples + /// + /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory + /// access, we may receive a buffer in shared memory from that device as follows: + /// + /// ```rust,no_run + /// #![feature(atomic_volatile)] + /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; + /// use std::ptr; + /// + /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); + /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); + /// + /// // Spin until we see a non-zero value. + /// let buf = 'buf: loop { + /// let buf = unsafe { atomic_ptr.load_volatile(Ordering::Relaxed) }; + /// if !buf.is_null() { + /// break 'buf buf; + /// } + /// }; + /// // Synchronize with the store whose value we just read. + /// // Note: a standard acquire fence may not be sufficient to synchronize with DMA devices. + /// // Depending on your target, you may have to use inline assembly to emit a special fence. + /// fence(Ordering::Acquire); + /// + /// // Now process the data in `buf`. #[inline] #[unstable(feature = "atomic_volatile", issue = "158947")] #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> bool { + pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> T { // SAFETY: follows from our own safety requirements. unsafe { - atomic_load::<_, /* VOLATILE */ true>(self.cast::(), order) != 0 + transmute_unchecked(atomic_load::<_, /* VOLATILE */ true>( + self.cast::(), + order, + )) } } - /// Stores a value into the bool. + /// Stores a value atomically. /// /// `store` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. @@ -829,19 +1085,23 @@ impl AtomicBool { /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn store(&self, val: bool, order: Ordering) { + pub const fn store(&self, val: T, order: Ordering) { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. unsafe { - atomic_store::<_, /* VOLATILE */ false>(self.v.get().cast::(), val as u8, order); + atomic_store::<_, /* VOLATILE */ false>( + self.v.get().cast::(), + transmute_unchecked::<_, T::OpType>(val), + order, + ); } } - /// Performs a volatile atomic store into the bool. + /// Performs a volatile atomic store. /// /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. @@ -862,54 +1122,75 @@ impl AtomicBool { /// # Panics /// /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// + /// # Examples + /// + /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory + /// access, we may submit a buffer in shared memory to that device as follows: + /// + /// ```rust,no_run + /// #![feature(atomic_volatile)] + /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; + /// use std::ptr; + /// + /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); + /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); + /// + /// // Prepare some data for the DMA device. + /// # fn get_dma_buffer() -> *mut u8 { panic!() } + /// let buf = get_dma_buffer(); + /// + /// // Ensure the other side can synchronize with the store we do below. + /// // Note: a standard release fence may not be sufficient to synchronize with DMA devices. + /// // Depending on your target, you may have to use inline assembly to emit a special fence. + /// fence(Ordering::Release); + /// + /// unsafe { atomic_ptr.store_volatile(buf, Ordering::Relaxed) }; + /// ``` #[inline] #[unstable(feature = "atomic_volatile", issue = "158947")] #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const unsafe fn store_volatile(self: *const Self, val: bool, order: Ordering) { + pub const unsafe fn store_volatile(self: *const Self, val: T, order: Ordering) { // SAFETY: follows from our own safety requirements. unsafe { - atomic_store::<_, /* VOLATILE */ true>(self.cast::().cast_mut(), val as u8, order); + atomic_store::<_, /* VOLATILE */ true>( + self.cast::().cast_mut(), + transmute_unchecked::<_, T::OpType>(val), + order, + ); } } +} - /// Stores a value into the bool, returning the previous value. +impl Atomic { + /// Stores a value into the pointer, returning the previous value. /// /// `swap` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let some_bool = AtomicBool::new(true); + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// assert_eq!(some_bool.swap(false, Ordering::Relaxed), true); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let some_atomic = AtomicU32::new(5); + /// let value = some_atomic.swap(10, Ordering::Relaxed); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn swap(&self, val: bool, order: Ordering) -> bool { - if EMULATE_ATOMIC_BOOL { - if val { self.fetch_or(true, order) } else { self.fetch_and(false, order) } - } else { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_swap(self.v.get().cast::(), val as u8, order) != 0 } - } + pub const fn swap(&self, v: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_swap(self.as_ptr(), v, order) } } - /// Stores a value into the [`bool`] if the current value is the same as the `current` value. + /// Stores a value into the pointer if the current value is the same as the `current` value. /// /// The return value is always the previous value. If it is equal to `current`, then the value /// was updated. @@ -920,8 +1201,6 @@ impl AtomicBool { /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it /// happens, and using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. /// /// # Migrating to `compare_exchange` and `compare_exchange_weak` /// @@ -949,34 +1228,26 @@ impl AtomicBool { /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let some_bool = AtomicBool::new(true); + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// assert_eq!(some_bool.compare_and_swap(true, false, Ordering::Relaxed), true); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let some_atomic = AtomicU32::new(5); /// - /// assert_eq!(some_bool.compare_and_swap(true, true, Ordering::Relaxed), false); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let value = some_atomic.compare_and_swap(5, 10, Ordering::Relaxed); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[deprecated( since = "1.50.0", note = "Use `compare_exchange` or `compare_exchange_weak` instead" )] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { - match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { - Ok(x) => x, - Err(x) => x, - } + pub fn compare_and_swap(&self, current: T, new: T, order: Ordering) -> T { + self.compare_exchange(current, new, order, strongest_failure_ordering(order)) + .unwrap_or_else(|x| x) } - /// Stores a value into the [`bool`] if the current value is the same as the `current` value. + /// Stores a value into the pointer if the current value is the same as the `current` value. /// /// The return value is a result indicating whether the new value was written and containing /// the previous value. On success this value is guaranteed to be equal to `current`. @@ -990,27 +1261,20 @@ impl AtomicBool { /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// operations on pointers. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{AtomicPtr, Ordering}; /// - /// let some_bool = AtomicBool::new(true); + /// let ptr = &mut 5; + /// let some_ptr = AtomicPtr::new(ptr); /// - /// assert_eq!(some_bool.compare_exchange(true, - /// false, - /// Ordering::Acquire, - /// Ordering::Relaxed), - /// Ok(true)); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let other_ptr = &mut 10; /// - /// assert_eq!(some_bool.compare_exchange(true, true, - /// Ordering::SeqCst, - /// Ordering::Acquire), - /// Err(false)); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let value = some_ptr.compare_exchange(ptr, other_ptr, + /// Ordering::SeqCst, Ordering::Relaxed); /// ``` /// /// # Considerations @@ -1020,70 +1284,31 @@ impl AtomicBool { /// `compare_exchange` with the previous load *does not ensure* that other threads have not /// changed the value in the interim. This is usually important when the *equality* check in /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the - /// [ABA problem]. + /// does not necessarily imply identity. This is a particularly common case for pointers, as + /// a pointer holding the same address does not imply that the same object exists at that + /// address! In this case, `compare_exchange` can lead to the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[doc(alias = "compare_and_swap")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub const fn compare_exchange( &self, - current: bool, - new: bool, + current: T, + new: T, success: Ordering, failure: Ordering, - ) -> Result { - if EMULATE_ATOMIC_BOOL { - // Pick the strongest ordering from success and failure. - let order = match (success, failure) { - (SeqCst, _) => SeqCst, - (_, SeqCst) => SeqCst, - (AcqRel, _) => AcqRel, - (_, AcqRel) => { - panic!("there is no such thing as an acquire-release failure ordering") - } - (Release, Acquire) => AcqRel, - (Acquire, _) => Acquire, - (_, Acquire) => Acquire, - (Release, Relaxed) => Release, - (_, Release) => panic!("there is no such thing as a release failure ordering"), - (Relaxed, Relaxed) => Relaxed, - }; - let old = if current == new { - // This is a no-op, but we still need to perform the operation - // for memory ordering reasons. - self.fetch_or(false, order) - } else { - // This sets the value to the new one and returns the old one. - self.swap(new, order) - }; - if old == current { Ok(old) } else { Err(old) } - } else { - // SAFETY: data races are prevented by atomic intrinsics. - match unsafe { - atomic_compare_exchange( - self.v.get().cast::(), - current as u8, - new as u8, - success, - failure, - ) - } { - Ok(x) => Ok(x != 0), - Err(x) => Err(x != 0), - } - } + ) -> Result { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_compare_exchange(self.as_ptr(), current, new, success, failure) } } - /// Stores a value into the [`bool`] if the current value is the same as the `current` value. + /// Stores a value into the pointer if the current value is the same as the `current` value. /// - /// Unlike [`AtomicBool::compare_exchange`], this function is allowed to spuriously fail even when the + /// Unlike [`AtomicPtr::compare_exchange`], this function is allowed to spuriously fail even when the /// comparison succeeds, which can result in more efficient code on some platforms. The /// return value is a result indicating whether the new value was written and containing the /// previous value. @@ -1096,20 +1321,16 @@ impl AtomicBool { /// of this operation [`Relaxed`], and using [`Release`] makes the successful load /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let val = AtomicBool::new(false); + /// let some_atomic = AtomicU32::new(5); /// - /// let new = true; - /// let mut old = val.load(Ordering::Relaxed); + /// let mut old = some_atomic.load(Ordering::Relaxed); /// loop { - /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { + /// match some_atomic.compare_exchange_weak(old, 10, Ordering::SeqCst, Ordering::Relaxed) { /// Ok(_) => break, /// Err(x) => old = x, /// } @@ -1123,1127 +1344,1084 @@ impl AtomicBool { /// `compare_exchange` with the previous load *does not ensure* that other threads have not /// changed the value in the interim. This is usually important when the *equality* check in /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the - /// [ABA problem]. + /// does not necessarily imply identity. This is a particularly common case for pointers, as + /// a pointer holding the same address does not imply that the same object exists at that + /// address! In this case, `compare_exchange` can lead to the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[doc(alias = "compare_and_swap")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub const fn compare_exchange_weak( &self, - current: bool, - new: bool, + current: T, + new: T, success: Ordering, failure: Ordering, - ) -> Result { - if EMULATE_ATOMIC_BOOL { - return self.compare_exchange(current, new, success, failure); - } - - // SAFETY: data races are prevented by atomic intrinsics. - match unsafe { - atomic_compare_exchange_weak( - self.v.get().cast::(), - current as u8, - new as u8, - success, - failure, - ) - } { - Ok(x) => Ok(x != 0), - Err(x) => Err(x != 0), - } + ) -> Result { + // SAFETY: This intrinsic is unsafe because it operates on a raw pointer + // but we know for sure that the pointer is valid (we just got it from + // an `UnsafeCell` that we have by reference) and the atomic operation + // itself allows us to safely mutate the `UnsafeCell` contents. + unsafe { atomic_compare_exchange_weak(self.as_ptr(), current, new, success, failure) } } - /// Logical "and" with a boolean value. - /// - /// Performs a logical "and" operation on the current value and the argument `val`, and sets - /// the new value to the result. + /// An alias for [`AtomicPtr::try_update`]. + #[inline] + #[stable(feature = "atomic_fetch_update", since = "1.53.0")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + #[deprecated( + since = "1.99.0", + note = "renamed to `try_update` for consistency", + suggestion = "try_update" + )] + pub fn fetch_update(&self, set_order: Ordering, fetch_order: Ordering, f: F) -> Result + where + F: FnMut(T) -> Option, + { + self.try_update(set_order, fetch_order, f) + } + + /// Fetches the value, and applies a function to it that returns an optional + /// new value. Returns a `Result` of `Ok(previous_value)` if the function + /// returned `Some(_)`, else `Err(previous_value)`. /// - /// Returns the previous value. + /// See also: [`update`](`Atomic::update`). /// - /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. + /// Note: This may call the function multiple times if the value has been + /// changed from other threads in the meantime, as long as the function + /// returns `Some(_)`, but the function will have been applied only once to + /// the stored value. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// `try_update` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. The first describes the required ordering for + /// when the operation finally succeeds while the second describes the + /// required ordering for loads. These correspond to the success and failure + /// orderings of [`Atomic::compare_exchange`] respectively. /// - /// # Examples + /// Using [`Acquire`] as success ordering makes the store part of this + /// operation [`Relaxed`], and using [`Release`] makes the final successful + /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], + /// [`Acquire`] or [`Relaxed`]. /// - /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// # Considerations /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// This method is not magic; it is not provided by the hardware, and does not act like a + /// critical section or mutex. /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_and(true, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], + /// which is a particularly common pitfall for pointers! /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// + /// # Examples + /// + /// ```rust + /// use std::sync::atomic::{AtomicU32, Ordering}; + /// + /// let some_atomic = AtomicU32::new(5); + /// + /// assert_eq!(some_atomic.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(5)); + /// let result = some_atomic.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| { + /// if x == 5 { + /// Some(10) + /// } else { + /// None + /// } + /// }); + /// assert_eq!(result, Ok(5)); + /// assert_eq!(some_atomic.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "atomic_try_update", since = "1.95.0")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_and(&self, val: bool, order: Ordering) -> bool { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_and(self.v.get().cast::(), val as u8, order) != 0 } + pub fn try_update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: impl FnMut(T) -> Option, + ) -> Result { + let mut prev = self.load(fetch_order); + while let Some(next) = f(prev) { + match self.compare_exchange_weak(prev, next, set_order, fetch_order) { + x @ Ok(_) => return x, + Err(next_prev) => prev = next_prev, + } + } + Err(prev) } - /// Logical "nand" with a boolean value. + /// Fetches the value, applies a function to it that it return a new value. + /// The new value is stored and the old value is returned. /// - /// Performs a logical "nand" operation on the current value and the argument `val`, and sets - /// the new value to the result. + /// See also: [`try_update`](`AtomicPtr::try_update`). /// - /// Returns the previous value. + /// Note: This may call the function multiple times if the value has been changed from other threads in + /// the meantime, but the function will have been applied only once to the stored value. /// - /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. + /// `update` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. The first describes the required ordering for + /// when the operation finally succeeds while the second describes the + /// required ordering for loads. These correspond to the success and failure + /// orderings of [`AtomicPtr::compare_exchange`] respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// operations on pointers. + /// + /// # Considerations + /// + /// This method is not magic; it is not provided by the hardware, and does not act like a + /// critical section or mutex. + /// + /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], + /// which is a particularly common pitfall for pointers! + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap /// /// # Examples /// - /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// ```rust /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_nand(true, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst) as usize, 0); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// let some_atomic = AtomicU32::new(5); /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// let result = some_atomic.update(Ordering::SeqCst, Ordering::SeqCst, |_| 10); + /// assert_eq!(result, 5); + /// assert_eq!(some_atomic.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "atomic_try_update", since = "1.95.0")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_nand(&self, val: bool, order: Ordering) -> bool { - // We can't use atomic_nand here because it can result in a bool with - // an invalid value. This happens because the atomic operation is done - // with an 8-bit integer internally, which would set the upper 7 bits. - // So we just use fetch_xor or swap instead. - if val { - // !(x & true) == !x - // We must invert the bool. - self.fetch_xor(true, order) - } else { - // !(x & false) == true - // We must set the bool to true. - self.swap(true, order) + pub fn update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: impl FnMut(T) -> T, + ) -> T { + let mut prev = self.load(fetch_order); + loop { + match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { + Ok(x) => break x, + Err(next_prev) => prev = next_prev, + } } } +} - /// Logical "or" with a boolean value. - /// - /// Performs a logical "or" operation on the current value and the argument `val`, and sets the - /// new value to the result. +impl Atomic { + /// Gets atomic access to a `&mut T`. /// - /// Returns the previous value. - /// - /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// **Note:** This function is only available on targets where `Atomic` has the same + /// alignment as `T`. /// /// # Examples /// /// ``` /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// let mut some_bool = true; + /// let a = AtomicBool::from_mut(&mut some_bool); + /// a.store(false, Ordering::Relaxed); + /// assert_eq!(some_bool, false); + /// ``` + #[inline] + #[cfg(target_has_atomic_primitive_alignment = "8")] + #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut(v: &mut T) -> &mut Self { + const { + assert!( + align_of::() == align_of::(), + "This function is only available if Atomic and T have the same alignment" + ); + } + + // SAFETY: + // * The mutable reference guarantees unique ownership + // * The contract of AtomicPrimitive requires that T::Storage has the same, or higher + // alignment as T, casting from a higher alignment is safe. + // * The contract of AtomicPrimitive requires that T::Storage has the same size as T + // * No atomic operations are capable of transforming the underlying data into an invalid + // variant of T. + unsafe { &mut *(v as *mut T as *mut Self) } + } + + /// Gets atomic access to a `&mut [T]` slice. /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_or(true, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// **Note:** This function is only available on targets where `Atomic` has the same + /// alignment as `T` /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// # Examples + /// + /// ```rust,ignore-wasm + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let mut some_bools = [false; 10]; + /// let a = &*AtomicBool::from_mut_slice(&mut some_bools); + /// std::thread::scope(|s| { + /// for i in 0..a.len() { + /// s.spawn(move || a[i].store(true, Ordering::Relaxed)); + /// } + /// }); + /// assert_eq!(some_bools, [true; 10]); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "atomic_from_mut", since = "1.98.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_or(&self, val: bool, order: Ordering) -> bool { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_or(self.v.get().cast::(), val as u8, order) != 0 } + pub const fn from_mut_slice(v: &mut [T]) -> &mut [Self] { + const { + assert!( + align_of::() == align_of::(), + "This function is only available if Atomic and T have the same alignment" + ); + } + + // SAFETY: + // * The mutable reference guarantees unique ownership + // * The alignment of both T and Self is the same. + // * Since the alignment and size of Self and T are the same, and they have the same + // invariants, they can be freely cast between one another. + unsafe { &mut *(v as *mut [T] as *mut [Self]) } } +} - /// Logical "xor" with a boolean value. - /// - /// Performs a logical "xor" operation on the current value and the argument `val`, and sets - /// the new value to the result. +impl Atomic { + /// Adds to the current value, returning the previous value. /// - /// Returns the previous value. + /// This operation wraps around on overflow. /// - /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// `fetch_add` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); - /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_xor(true, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// let foo = AtomicU32::new(0); + /// assert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_xor(&self, val: bool, order: Ordering) -> bool { + pub const fn fetch_add(&self, val: T, order: Ordering) -> T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_xor(self.v.get().cast::(), val as u8, order) != 0 } + unsafe { atomic_add(self.as_ptr(), val, order) } } - /// Logical "not" with a boolean value. - /// - /// Performs a logical "not" operation on the current value, and sets - /// the new value to the result. + /// Subtracts from the current value, returning the previous value. /// - /// Returns the previous value. + /// This operation wraps around on overflow. /// - /// `fetch_not` takes an [`Ordering`] argument which describes the memory ordering + /// `fetch_sub` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_not(Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_not(Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// let foo = AtomicU32::new(20); + /// assert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "atomic_bool_fetch_not", since = "1.81.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_not(&self, order: Ordering) -> bool { - self.fetch_xor(true, order) + pub const fn fetch_sub(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_sub(self.as_ptr(), val, order) } } - /// Returns a mutable pointer to the underlying [`bool`]. + /// Maximum with the current value. /// - /// Doing non-atomic reads and writes on the resulting boolean can be a data race. - /// This method is mostly useful for FFI, where the function signature may use - /// `*mut bool` instead of `&AtomicBool`. + /// Finds the maximum of the current value and the argument `val`, and + /// sets the new value to the result. /// - /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the - /// atomic types work with interior mutability. All modifications of an atomic change the value - /// through a shared reference, and can do so safely as long as they use atomic operations. Any - /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the - /// requirements of the [memory model]. + /// Returns the previous value. + /// + /// `fetch_max` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// # Examples /// - /// ```ignore (extern-declaration) - /// # fn main() { - /// use std::sync::atomic::AtomicBool; + /// ``` + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// extern "C" { - /// fn my_atomic_op(arg: *mut bool); - /// } + /// let foo = AtomicU32::new(23); + /// assert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23); + /// assert_eq!(foo.load(Ordering::SeqCst), 42); + /// ``` + /// + /// If you want to obtain the maximum value in one step, you can use the following: /// - /// let mut atomic = AtomicBool::new(true); - /// unsafe { - /// my_atomic_op(atomic.as_ptr()); - /// } - /// # } /// ``` + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// [memory model]: self#memory-model-for-atomic-accesses - #[inline] - #[stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_never_returns_null_ptr] - #[rustc_should_not_be_called_on_const_items] - pub const fn as_ptr(&self) -> *mut bool { - self.v.get().cast() - } - - /// An alias for [`AtomicBool::try_update`]. + /// let foo = AtomicU32::new(23); + /// let bar = 42; + /// let max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar); + /// assert!(max_foo == 42); + /// ``` #[inline] - #[stable(feature = "atomic_fetch_update", since = "1.53.0")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - #[deprecated( - since = "1.99.0", - note = "renamed to `try_update` for consistency", - suggestion = "try_update" - )] - pub fn fetch_update( - &self, - set_order: Ordering, - fetch_order: Ordering, - f: F, - ) -> Result - where - F: FnMut(bool) -> Option, - { - self.try_update(set_order, fetch_order, f) + pub const fn fetch_max(&self, val: T, order: Ordering) -> T { + if T::IS_SIGNED { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_max(self.as_ptr(), val, order) } + } else { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_umax(self.as_ptr(), val, order) } + } } - /// Fetches the value, and applies a function to it that returns an optional - /// new value. Returns a `Result` of `Ok(previous_value)` if the function - /// returned `Some(_)`, else `Err(previous_value)`. - /// - /// See also: [`update`](`AtomicBool::update`). - /// - /// Note: This may call the function multiple times if the value has been - /// changed from other threads in the meantime, as long as the function - /// returns `Some(_)`, but the function will have been applied only once to - /// the stored value. - /// - /// `try_update` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering for - /// when the operation finally succeeds while the second describes the - /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicBool::compare_exchange`] respectively. + /// Minimum with the current value. /// - /// Using [`Acquire`] as success ordering makes the store part of this - /// operation [`Relaxed`], and using [`Release`] makes the final successful - /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], - /// [`Acquire`] or [`Relaxed`]. + /// Finds the minimum of the current value and the argument `val`, and + /// sets the new value to the result. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// Returns the previous value. /// - /// # Considerations + /// `fetch_min` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// This method is not magic; it is not provided by the hardware, and does not act like a - /// critical section or mutex. + /// # Examples /// - /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. + /// ``` + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// let foo = AtomicU32::new(23); + /// assert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 23); + /// assert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 22); + /// ``` /// - /// # Examples + /// If you want to obtain the minimum value in one step, you can use the following: /// - /// ```rust - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// ``` + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let x = AtomicBool::new(false); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(false)); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(false)); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(true)); - /// assert_eq!(x.load(Ordering::SeqCst), false); + /// let foo = AtomicU32::new(23); + /// let bar = 12; + /// let min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar); + /// assert_eq!(min_foo, 12); /// ``` #[inline] - #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn try_update( - &self, - set_order: Ordering, - fetch_order: Ordering, - mut f: impl FnMut(bool) -> Option, - ) -> Result { - let mut prev = self.load(fetch_order); - while let Some(next) = f(prev) { - match self.compare_exchange_weak(prev, next, set_order, fetch_order) { - x @ Ok(_) => return x, - Err(next_prev) => prev = next_prev, - } + pub const fn fetch_min(&self, val: T, order: Ordering) -> T { + if ::IS_SIGNED { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_min(self.as_ptr(), val, order) } + } else { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_umin(self.as_ptr(), val, order) } } - Err(prev) } +} - /// Fetches the value, applies a function to it that it return a new value. - /// The new value is stored and the old value is returned. +impl Atomic { + /// Bitwise "nand" with the current value. /// - /// See also: [`try_update`](`AtomicBool::try_update`). + /// Performs a bitwise "nand" operation on the current value and the argument `val`, and + /// sets the new value to the result. /// - /// Note: This may call the function multiple times if the value has been changed from other threads in - /// the meantime, but the function will have been applied only once to the stored value. + /// Returns the previous value. /// - /// `update` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering for - /// when the operation finally succeeds while the second describes the - /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicBool::compare_exchange`] respectively. + /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load - /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// # Examples /// - /// **Note:** This method is only available on platforms that support atomic operations on `u8`. + /// ``` + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// # Considerations + /// let foo = AtomicU32::new(0x13); + /// assert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13); + /// assert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31)); + /// ``` + #[inline] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_nand(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_nand(self.as_ptr(), val, order) } + } + + /// Bitwise "and" with the current value. /// - /// This method is not magic; it is not provided by the hardware, and does not act like a - /// critical section or mutex. + /// Performs a bitwise "and" operation on the current value and the argument `val`, and + /// sets the new value to the result. /// - /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. + /// Returns the previous value. /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// # Examples /// - /// ```rust - /// - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// ``` + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let x = AtomicBool::new(false); - /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), false); - /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), true); - /// assert_eq!(x.load(Ordering::SeqCst), false); + /// let foo = AtomicU32::new(0b101101); + /// assert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b100001); /// ``` #[inline] - #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn update( - &self, - set_order: Ordering, - fetch_order: Ordering, - mut f: impl FnMut(bool) -> bool, - ) -> bool { - let mut prev = self.load(fetch_order); - loop { - match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { - Ok(x) => break x, - Err(next_prev) => prev = next_prev, - } - } + pub const fn fetch_and(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_and(self.as_ptr(), val, order) } } -} -#[cfg(target_has_atomic_load_store = "ptr")] -impl AtomicPtr { - /// Creates a new `AtomicPtr`. + /// Bitwise "or" with the current value. + /// + /// Performs a bitwise "or" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// # Examples /// /// ``` - /// use std::sync::atomic::AtomicPtr; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let ptr = &mut 5; - /// let atomic_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicU32::new(0b101101); + /// assert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b111111); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_atomic_new", since = "1.24.0")] - pub const fn new(p: *mut T) -> AtomicPtr { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(p) } + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_or(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_or(self.as_ptr(), val, order) } } - /// Creates a new `AtomicPtr` from a pointer. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::{self, AtomicPtr}; + /// Bitwise "xor" with the current value. /// - /// // Get a pointer to an allocated value - /// let ptr: *mut *mut u8 = Box::into_raw(Box::new(std::ptr::null_mut())); - /// - /// assert!(ptr.cast::>().is_aligned()); + /// Performs a bitwise "xor" operation on the current value and the argument `val`, and + /// sets the new value to the result. /// - /// { - /// // Create an atomic view of the allocated value - /// let atomic = unsafe { AtomicPtr::from_ptr(ptr) }; + /// Returns the previous value. /// - /// // Use `atomic` for atomic operations, possibly share it with other threads - /// atomic.store(std::ptr::NonNull::dangling().as_ptr(), atomic::Ordering::Relaxed); - /// } + /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// // It's ok to non-atomically access the value behind `ptr`, - /// // since the reference to the atomic ended its lifetime in the block above - /// assert!(!unsafe { *ptr }.is_null()); + /// # Examples /// - /// // Deallocate the value - /// unsafe { drop(Box::from_raw(ptr)) } /// ``` + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// # Safety - /// - /// * `ptr` must be aligned to `align_of::>()` (note that on some platforms this - /// can be bigger than `align_of::<*mut T>()`). - /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`. - /// * You must adhere to the [Memory model for atomic accesses]. In particular, it is not - /// allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different - /// sizes, without synchronization. - /// - /// [valid]: crate::ptr#safety - /// [Memory model for atomic accesses]: self#memory-model-for-atomic-accesses + /// let foo = AtomicU32::new(0b101101); + /// assert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b011110); + /// ``` #[inline] - #[stable(feature = "atomic_from_ptr", since = "1.75.0")] - #[rustc_const_stable(feature = "const_atomic_from_ptr", since = "1.84.0")] - pub const unsafe fn from_ptr<'a>(ptr: *mut *mut T) -> &'a AtomicPtr { - // SAFETY: guaranteed by the caller - unsafe { &*ptr.cast() } - } - - /// Creates a new pointer to `AtomicPtr` from a pointer. - /// - /// This is useful if you want to do volatile atomic accesses, and thus avoid creating - /// a reference to the destination. - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - pub const fn from_ptr_raw(ptr: *mut *mut T) -> *const AtomicPtr { - ptr.cast_const().cast() + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_xor(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_xor(self.as_ptr(), val, order) } } +} - /// Creates a new `AtomicPtr` initialized with a null pointer. - /// - /// # Examples - /// - /// ``` - /// #![feature(atomic_ptr_null)] - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let atomic_ptr = AtomicPtr::<()>::null(); - /// assert!(atomic_ptr.load(Ordering::Relaxed).is_null()); - /// ``` +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl Default for Atomic { #[inline] - #[must_use] - #[unstable(feature = "atomic_ptr_null", issue = "150733")] - pub const fn null() -> AtomicPtr { - AtomicPtr::new(crate::ptr::null_mut()) + fn default() -> Self { + Self::new(T::default()) } +} - /// Returns a mutable reference to the underlying pointer. - /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let mut data = 10; - /// let mut atomic_ptr = AtomicPtr::new(&mut data); - /// let mut other_data = 5; - /// *atomic_ptr.get_mut() = &mut other_data; - /// assert_eq!(unsafe { *atomic_ptr.load(Ordering::SeqCst) }, 5); - /// ``` - #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut(&mut self) -> &mut *mut T { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { &mut *self.as_ptr() } +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +#[rustc_const_unstable(feature = "const_convert", issue = "143773")] +impl From for Atomic { + fn from(value: T) -> Self { + Self::new(value) } +} + +/// An [`AtomicBool`] initialized to `false`. +#[cfg(target_has_atomic_load_store = "8")] +#[stable(feature = "rust1", since = "1.0.0")] +#[deprecated( + since = "1.34.0", + note = "the `new` function is now preferred", + suggestion = "AtomicBool::new(false)" +)] +#[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] +pub const ATOMIC_BOOL_INIT: AtomicBool = AtomicBool::new(false); - /// Gets atomic access to a pointer. +#[cfg(target_has_atomic_load_store = "8")] +impl AtomicBool { + /// Stores a value into the bool, returning the previous value. + /// + /// `swap` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This function is only available on targets where `AtomicPtr` has the same alignment as `*const T` + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let some_bool = AtomicBool::new(true); /// - /// let mut data = 123; - /// let mut some_ptr = &mut data as *mut i32; - /// let a = AtomicPtr::from_mut(&mut some_ptr); - /// let mut other_data = 456; - /// a.store(&mut other_data, Ordering::Relaxed); - /// assert_eq!(unsafe { *some_ptr }, 456); + /// assert_eq!(some_bool.swap(false, Ordering::Relaxed), true); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` #[inline] - #[cfg(target_has_atomic_primitive_alignment = "ptr")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut(v: &mut *mut T) -> &mut Self { - let [] = [(); align_of::>() - align_of::<*mut ()>()]; - // SAFETY: - // - the mutable reference guarantees unique ownership. - // - the alignment of `*mut T` and `Self` is the same on all platforms - // supported by rust, as verified above. - unsafe { &mut *(v as *mut *mut T as *mut Self) } + #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn swap(&self, val: bool, order: Ordering) -> bool { + if EMULATE_ATOMIC_BOOL { + if val { self.fetch_or(true, order) } else { self.fetch_and(false, order) } + } else { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_swap(self.v.get().cast::(), val as u8, order) != 0 } + } } - /// Gets non-atomic access to a `&mut [AtomicPtr]` slice. + /// Stores a value into the [`bool`] if the current value is the same as the `current` value. /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. + /// The return value is always the previous value. If it is equal to `current`, then the value + /// was updated. /// - /// # Examples + /// `compare_and_swap` also takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. Notice that even when using [`AcqRel`], the operation + /// might fail and hence just perform an `Acquire` load, but not have `Release` semantics. + /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it + /// happens, and using [`Release`] makes the load part [`Relaxed`]. /// - /// ```ignore-wasm - /// use std::ptr::null_mut; - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// - /// let mut some_ptrs = [const { AtomicPtr::new(null_mut::()) }; 10]; + /// # Migrating to `compare_exchange` and `compare_exchange_weak` /// - /// let view: &mut [*mut String] = AtomicPtr::get_mut_slice(&mut some_ptrs); - /// assert_eq!(view, [null_mut::(); 10]); - /// view - /// .iter_mut() - /// .enumerate() - /// .for_each(|(i, ptr)| *ptr = Box::into_raw(Box::new(format!("iteration#{i}")))); + /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for + /// memory orderings: /// - /// std::thread::scope(|s| { - /// for ptr in &some_ptrs { - /// s.spawn(move || { - /// let ptr = ptr.load(Ordering::Relaxed); - /// assert!(!ptr.is_null()); - /// - /// let name = unsafe { Box::from_raw(ptr) }; - /// println!("Hello, {name}!"); - /// }); - /// } - /// }); - /// ``` - #[inline] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut_slice(this: &mut [Self]) -> &mut [*mut T] { - // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *(this as *mut [Self] as *mut [*mut T]) } - } - - /// Gets atomic access to a slice of pointers. + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst /// - /// **Note:** This function is only available on targets where `AtomicPtr` has the same alignment as `*const T` + /// `compare_and_swap` and `compare_exchange` also differ in their return type. You can use + /// `compare_exchange(...).unwrap_or_else(|x| x)` to recover the behavior of `compare_and_swap`, + /// but in most cases it is more idiomatic to check whether the return value is `Ok` or `Err` + /// rather than to infer success vs failure based on the value that was read. /// - /// # Examples + /// During migration, consider whether it makes sense to use `compare_exchange_weak` instead. + /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, + /// which allows the compiler to generate better assembly code when the compare and swap + /// is used in a loop. /// - /// ```ignore-wasm - /// use std::ptr::null_mut; - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// # Examples /// - /// let mut some_ptrs = [null_mut::(); 10]; - /// let a = &*AtomicPtr::from_mut_slice(&mut some_ptrs); - /// std::thread::scope(|s| { - /// for i in 0..a.len() { - /// s.spawn(move || { - /// let name = Box::new(format!("thread{i}")); - /// a[i].store(Box::into_raw(name), Ordering::Relaxed); - /// }); - /// } - /// }); - /// for p in some_ptrs { - /// assert!(!p.is_null()); - /// let name = unsafe { Box::from_raw(p) }; - /// println!("Hello, {name}!"); - /// } /// ``` - #[inline] - #[cfg(target_has_atomic_primitive_alignment = "ptr")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut_slice(v: &mut [*mut T]) -> &mut [Self] { - // SAFETY: - // - the mutable reference guarantees unique ownership. - // - the alignment of `*mut T` and `Self` is the same on all platforms - // supported by rust, as verified above. - unsafe { &mut *(v as *mut [*mut T] as *mut [Self]) } - } - - /// Consumes the atomic and returns the contained value. - /// - /// This is safe because passing `self` by value guarantees that no other threads are - /// concurrently accessing the atomic data. + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// # Examples + /// let some_bool = AtomicBool::new(true); /// - /// ``` - /// use std::sync::atomic::AtomicPtr; + /// assert_eq!(some_bool.compare_and_swap(true, false, Ordering::Relaxed), true); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// - /// let mut data = 5; - /// let atomic_ptr = AtomicPtr::new(&mut data); - /// assert_eq!(unsafe { *atomic_ptr.into_inner() }, 5); + /// assert_eq!(some_bool.compare_and_swap(true, true, Ordering::Relaxed), false); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0")] - pub const fn into_inner(self) -> *mut T { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(self) } + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[deprecated( + since = "1.50.0", + note = "Use `compare_exchange` or `compare_exchange_weak` instead" + )] + #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { + match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { + Ok(x) => x, + Err(x) => x, + } } - /// Loads a value from the pointer. + /// Stores a value into the [`bool`] if the current value is the same as the `current` value. /// - /// `load` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// The return value is a result indicating whether the new value was written and containing + /// the previous value. On success this value is guaranteed to be equal to `current`. /// - /// # Panics + /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// Panics if `order` is [`Release`] or [`AcqRel`]. + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let some_bool = AtomicBool::new(true); + /// + /// assert_eq!(some_bool.compare_exchange(true, + /// false, + /// Ordering::Acquire, + /// Ordering::Relaxed), + /// Ok(true)); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// - /// let value = some_ptr.load(Ordering::Relaxed); + /// assert_eq!(some_bool.compare_exchange(true, true, + /// Ordering::SeqCst, + /// Ordering::Acquire), + /// Err(false)); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` + /// + /// # Considerations + /// + /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides + /// of CAS operations. In particular, a load of the value followed by a successful + /// `compare_exchange` with the previous load *does not ensure* that other threads have not + /// changed the value in the interim. This is usually important when the *equality* check in + /// the `compare_exchange` is being used to check the *identity* of a value, but equality + /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the + /// [ABA problem]. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[doc(alias = "compare_and_swap")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const fn load(&self, order: Ordering) -> *mut T { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { - atomic_load::<_, /* VOLATILE */ false>(self.as_ptr(), order) + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_exchange( + &self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering, + ) -> Result { + if EMULATE_ATOMIC_BOOL { + // Pick the strongest ordering from success and failure. + let order = match (success, failure) { + (SeqCst, _) => SeqCst, + (_, SeqCst) => SeqCst, + (AcqRel, _) => AcqRel, + (_, AcqRel) => { + panic!("there is no such thing as an acquire-release failure ordering") + } + (Release, Acquire) => AcqRel, + (Acquire, _) => Acquire, + (_, Acquire) => Acquire, + (Release, Relaxed) => Release, + (_, Release) => panic!("there is no such thing as a release failure ordering"), + (Relaxed, Relaxed) => Relaxed, + }; + let old = if current == new { + // This is a no-op, but we still need to perform the operation + // for memory ordering reasons. + self.fetch_or(false, order) + } else { + // This sets the value to the new one and returns the old one. + self.swap(new, order) + }; + if old == current { Ok(old) } else { Err(old) } + } else { + // SAFETY: data races are prevented by atomic intrinsics. + match unsafe { + atomic_compare_exchange( + self.v.get().cast::(), + current as u8, + new as u8, + success, + failure, + ) + } { + Ok(x) => Ok(x != 0), + Err(x) => Err(x != 0), + } } } - /// Perform a volatile atomic load from the pointer. - /// - /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. - /// - #[doc = include_str!("./atomic_load_volatile.md")] - /// - /// # Safety - /// - /// Behavior is undefined if any of the following conditions are violated: - /// - /// * `self` must be [valid] for reads, or `self` must point to memory - /// outside of all Rust allocations and reading from that memory must: - /// - not trap, and - /// - not cause any memory inside a Rust allocation to be modified. - /// - /// * `self` must be aligned to `align_of::>()` (note that on some platforms this - /// can be bigger than `align_of::<*mut T>()`). - /// - /// * Reading from `self` must produce a properly initialized value of type `*mut T`. + /// Stores a value into the [`bool`] if the current value is the same as the `current` value. /// - /// [valid]: core::ptr#safety + /// Unlike [`AtomicBool::compare_exchange`], this function is allowed to spuriously fail even when the + /// comparison succeeds, which can result in more efficient code on some platforms. The + /// return value is a result indicating whether the new value was written and containing the + /// previous value. /// - /// # Panics + /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// Panics if `order` is [`Release`] or [`AcqRel`]. + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// - /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory - /// access, we may receive a buffer in shared memory from that device as follows: - /// - /// ```rust,no_run - /// #![feature(atomic_volatile)] - /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; - /// use std::ptr; + /// ``` + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); - /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); + /// let val = AtomicBool::new(false); /// - /// // Spin until we see a non-zero value. - /// let buf = 'buf: loop { - /// let buf = unsafe { atomic_ptr.load_volatile(Ordering::Relaxed) }; - /// if !buf.is_null() { - /// break 'buf buf; + /// let new = true; + /// let mut old = val.load(Ordering::Relaxed); + /// loop { + /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { + /// Ok(_) => break, + /// Err(x) => old = x, /// } - /// }; - /// // Synchronize with the store whose value we just read. - /// // Note: a standard acquire fence may not be sufficient to synchronize with DMA devices. - /// // Depending on your target, you may have to use inline assembly to emit a special fence. - /// fence(Ordering::Acquire); - /// - /// // Now process the data in `buf`. + /// } /// ``` + /// + /// # Considerations + /// + /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides + /// of CAS operations. In particular, a load of the value followed by a successful + /// `compare_exchange` with the previous load *does not ensure* that other threads have not + /// changed the value in the interim. This is usually important when the *equality* check in + /// the `compare_exchange` is being used to check the *identity* of a value, but equality + /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the + /// [ABA problem]. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[doc(alias = "compare_and_swap")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> *mut T { - // SAFETY: follows from our own safety requirements. - unsafe { - atomic_load::<_, /* VOLATILE */ true>(self.cast::<*mut T>(), order) + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_exchange_weak( + &self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering, + ) -> Result { + if EMULATE_ATOMIC_BOOL { + return self.compare_exchange(current, new, success, failure); + } + + // SAFETY: data races are prevented by atomic intrinsics. + match unsafe { + atomic_compare_exchange_weak( + self.v.get().cast::(), + current as u8, + new as u8, + success, + failure, + ) + } { + Ok(x) => Ok(x != 0), + Err(x) => Err(x != 0), } } - /// Stores a value into the pointer. + /// Logical "and" with a boolean value. + /// + /// Performs a logical "and" operation on the current value and the argument `val`, and sets + /// the new value to the result. /// - /// `store` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. + /// Returns the previous value. /// - /// # Panics + /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// let other_ptr = &mut 10; + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_and(true, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// some_ptr.store(other_ptr, Ordering::Relaxed); + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn store(&self, ptr: *mut T, order: Ordering) { + pub const fn fetch_and(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { - atomic_store::<_, /* VOLATILE */ false>(self.as_ptr(), ptr, order); - } + unsafe { atomic_and(self.v.get().cast::(), val as u8, order) != 0 } } - /// Performs a volatile atomic store into the pointer. - /// - /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. - /// - #[doc = include_str!("./atomic_store_volatile.md")] - /// - /// # Safety - /// - /// Behavior is undefined if any of the following conditions are violated: - /// - /// * `self` must be either [valid] for writes, or `self` must point to memory - /// outside of all Rust allocations and writing to that memory must: - /// - not trap, and - /// - not cause any memory inside a Rust allocation to be modified. - /// - /// * `self` must be aligned to `align_of::>()` (note that on some platforms this - /// can be bigger than `align_of::<*mut T>()`). - /// - /// [valid]: core::ptr#safety - /// - /// # Panics - /// - /// Panics if `order` is [`Acquire`] or [`AcqRel`]. - /// - /// # Examples - /// - /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory - /// access, we may submit a buffer in shared memory to that device as follows: - /// - /// ```rust,no_run - /// #![feature(atomic_volatile)] - /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; - /// use std::ptr; - /// - /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); - /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); - /// - /// // Prepare some data for the DMA device. - /// # fn get_dma_buffer() -> *mut u8 { panic!() } - /// let buf = get_dma_buffer(); + /// Logical "nand" with a boolean value. /// - /// // Ensure the other side can synchronize with the store we do below. - /// // Note: a standard release fence may not be sufficient to synchronize with DMA devices. - /// // Depending on your target, you may have to use inline assembly to emit a special fence. - /// fence(Ordering::Release); + /// Performs a logical "nand" operation on the current value and the argument `val`, and sets + /// the new value to the result. /// - /// unsafe { atomic_ptr.store_volatile(buf, Ordering::Relaxed) }; - /// ``` - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const unsafe fn store_volatile(self: *const Self, ptr: *mut T, order: Ordering) { - // SAFETY: follows from our own safety requirements. - unsafe { - atomic_store::<_, /* VOLATILE */ true>(self.cast::<*mut T>().cast_mut(), ptr, order); - } - } - - /// Stores a value into the pointer, returning the previous value. + /// Returns the previous value. /// - /// `swap` takes an [`Ordering`] argument which describes the memory ordering + /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let other_ptr = &mut 10; + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_nand(true, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst) as usize, 0); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// let value = some_ptr.swap(other_ptr, Ordering::Relaxed); + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_swap(self.as_ptr(), ptr, order) } + pub const fn fetch_nand(&self, val: bool, order: Ordering) -> bool { + // We can't use atomic_nand here because it can result in a bool with + // an invalid value. This happens because the atomic operation is done + // with an 8-bit integer internally, which would set the upper 7 bits. + // So we just use fetch_xor or swap instead. + if val { + // !(x & true) == !x + // We must invert the bool. + self.fetch_xor(true, order) + } else { + // !(x & false) == true + // We must set the bool to true. + self.swap(true, order) + } } - /// Stores a value into the pointer if the current value is the same as the `current` value. - /// - /// The return value is always the previous value. If it is equal to `current`, then the value - /// was updated. - /// - /// `compare_and_swap` also takes an [`Ordering`] argument which describes the memory - /// ordering of this operation. Notice that even when using [`AcqRel`], the operation - /// might fail and hence just perform an `Acquire` load, but not have `Release` semantics. - /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it - /// happens, and using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. - /// - /// # Migrating to `compare_exchange` and `compare_exchange_weak` + /// Logical "or" with a boolean value. /// - /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for - /// memory orderings: + /// Performs a logical "or" operation on the current value and the argument `val`, and sets the + /// new value to the result. /// - /// Original | Success | Failure - /// -------- | ------- | ------- - /// Relaxed | Relaxed | Relaxed - /// Acquire | Acquire | Acquire - /// Release | Release | Relaxed - /// AcqRel | AcqRel | Acquire - /// SeqCst | SeqCst | SeqCst + /// Returns the previous value. /// - /// `compare_and_swap` and `compare_exchange` also differ in their return type. You can use - /// `compare_exchange(...).unwrap_or_else(|x| x)` to recover the behavior of `compare_and_swap`, - /// but in most cases it is more idiomatic to check whether the return value is `Ok` or `Err` - /// rather than to infer success vs failure based on the value that was read. + /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// During migration, consider whether it makes sense to use `compare_exchange_weak` instead. - /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, - /// which allows the compiler to generate better assembly code when the compare and swap - /// is used in a loop. + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let other_ptr = &mut 10; + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_or(true, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let value = some_ptr.compare_and_swap(ptr, other_ptr, Ordering::Relaxed); + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[deprecated( - since = "1.50.0", - note = "Use `compare_exchange` or `compare_exchange_weak` instead" - )] - #[cfg(target_has_atomic = "ptr")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_and_swap(&self, current: *mut T, new: *mut T, order: Ordering) -> *mut T { - match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { - Ok(x) => x, - Err(x) => x, - } + pub const fn fetch_or(&self, val: bool, order: Ordering) -> bool { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_or(self.v.get().cast::(), val as u8, order) != 0 } } - /// Stores a value into the pointer if the current value is the same as the `current` value. + /// Logical "xor" with a boolean value. /// - /// The return value is a result indicating whether the new value was written and containing - /// the previous value. On success this value is guaranteed to be equal to `current`. + /// Performs a logical "xor" operation on the current value and the argument `val`, and sets + /// the new value to the result. /// - /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. `success` describes the required ordering for the - /// read-modify-write operation that takes place if the comparison with `current` succeeds. - /// `failure` describes the required ordering for the load operation that takes place when - /// the comparison fails. Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// Returns the previous value. + /// + /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let other_ptr = &mut 10; + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_xor(true, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// let value = some_ptr.compare_exchange(ptr, other_ptr, - /// Ordering::SeqCst, Ordering::Relaxed); + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// ``` - /// - /// # Considerations - /// - /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides - /// of CAS operations. In particular, a load of the value followed by a successful - /// `compare_exchange` with the previous load *does not ensure* that other threads have not - /// changed the value in the interim. This is usually important when the *equality* check in - /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. This is a particularly common case for pointers, as - /// a pointer holding the same address does not imply that the same object exists at that - /// address! In this case, `compare_exchange` can lead to the [ABA problem]. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] - #[cfg(target_has_atomic = "ptr")] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange( - &self, - current: *mut T, - new: *mut T, - success: Ordering, - failure: Ordering, - ) -> Result<*mut T, *mut T> { + pub const fn fetch_xor(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_compare_exchange(self.as_ptr(), current, new, success, failure) } + unsafe { atomic_xor(self.v.get().cast::(), val as u8, order) != 0 } } - /// Stores a value into the pointer if the current value is the same as the `current` value. + /// Logical "not" with a boolean value. /// - /// Unlike [`AtomicPtr::compare_exchange`], this function is allowed to spuriously fail even when the - /// comparison succeeds, which can result in more efficient code on some platforms. The - /// return value is a result indicating whether the new value was written and containing the - /// previous value. + /// Performs a logical "not" operation on the current value, and sets + /// the new value to the result. /// - /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. `success` describes the required ordering for the - /// read-modify-write operation that takes place if the comparison with `current` succeeds. - /// `failure` describes the required ordering for the load operation that takes place when - /// the comparison fails. Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// Returns the previous value. + /// + /// `fetch_not` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let some_ptr = AtomicPtr::new(&mut 5); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_not(Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// let new = &mut 10; - /// let mut old = some_ptr.load(Ordering::Relaxed); - /// loop { - /// match some_ptr.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { - /// Ok(_) => break, - /// Err(x) => old = x, - /// } - /// } + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_not(Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// ``` - /// - /// # Considerations - /// - /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides - /// of CAS operations. In particular, a load of the value followed by a successful - /// `compare_exchange` with the previous load *does not ensure* that other threads have not - /// changed the value in the interim. This is usually important when the *equality* check in - /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. This is a particularly common case for pointers, as - /// a pointer holding the same address does not imply that the same object exists at that - /// address! In this case, `compare_exchange` can lead to the [ABA problem]. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] - #[cfg(target_has_atomic = "ptr")] + #[stable(feature = "atomic_bool_fetch_not", since = "1.81.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange_weak( - &self, - current: *mut T, - new: *mut T, - success: Ordering, - failure: Ordering, - ) -> Result<*mut T, *mut T> { - // SAFETY: This intrinsic is unsafe because it operates on a raw pointer - // but we know for sure that the pointer is valid (we just got it from - // an `UnsafeCell` that we have by reference) and the atomic operation - // itself allows us to safely mutate the `UnsafeCell` contents. - unsafe { atomic_compare_exchange_weak(self.as_ptr(), current, new, success, failure) } + pub const fn fetch_not(&self, order: Ordering) -> bool { + self.fetch_xor(true, order) } - /// An alias for [`AtomicPtr::try_update`]. + /// An alias for [`AtomicBool::try_update`]. #[inline] #[stable(feature = "atomic_fetch_update", since = "1.53.0")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] #[deprecated( @@ -2256,17 +2434,18 @@ impl AtomicPtr { set_order: Ordering, fetch_order: Ordering, f: F, - ) -> Result<*mut T, *mut T> + ) -> Result where - F: FnMut(*mut T) -> Option<*mut T>, + F: FnMut(bool) -> Option, { self.try_update(set_order, fetch_order, f) } + /// Fetches the value, and applies a function to it that returns an optional /// new value. Returns a `Result` of `Ok(previous_value)` if the function /// returned `Some(_)`, else `Err(previous_value)`. /// - /// See also: [`update`](`AtomicPtr::update`). + /// See also: [`update`](`AtomicBool::update`). /// /// Note: This may call the function multiple times if the value has been /// changed from other threads in the meantime, as long as the function @@ -2277,7 +2456,7 @@ impl AtomicPtr { /// ordering of this operation. The first describes the required ordering for /// when the operation finally succeeds while the second describes the /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicPtr::compare_exchange`] respectively. + /// orderings of [`AtomicBool::compare_exchange`] respectively. /// /// Using [`Acquire`] as success ordering makes the store part of this /// operation [`Relaxed`], and using [`Release`] makes the final successful @@ -2285,7 +2464,7 @@ impl AtomicPtr { /// [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Considerations /// @@ -2293,8 +2472,7 @@ impl AtomicPtr { /// critical section or mutex. /// /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], - /// which is a particularly common pitfall for pointers! + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap @@ -2302,34 +2480,25 @@ impl AtomicPtr { /// # Examples /// /// ```rust - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let ptr: *mut _ = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let new: *mut _ = &mut 10; - /// assert_eq!(some_ptr.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(ptr)); - /// let result = some_ptr.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| { - /// if x == ptr { - /// Some(new) - /// } else { - /// None - /// } - /// }); - /// assert_eq!(result, Ok(ptr)); - /// assert_eq!(some_ptr.load(Ordering::SeqCst), new); + /// let x = AtomicBool::new(false); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(false)); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(false)); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(true)); + /// assert_eq!(x.load(Ordering::SeqCst), false); /// ``` #[inline] #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub fn try_update( &self, set_order: Ordering, fetch_order: Ordering, - mut f: impl FnMut(*mut T) -> Option<*mut T>, - ) -> Result<*mut T, *mut T> { + mut f: impl FnMut(bool) -> Option, + ) -> Result { let mut prev = self.load(fetch_order); while let Some(next) = f(prev) { match self.compare_exchange_weak(prev, next, set_order, fetch_order) { @@ -2343,7 +2512,7 @@ impl AtomicPtr { /// Fetches the value, applies a function to it that it return a new value. /// The new value is stored and the old value is returned. /// - /// See also: [`try_update`](`AtomicPtr::try_update`). + /// See also: [`try_update`](`AtomicBool::try_update`). /// /// Note: This may call the function multiple times if the value has been changed from other threads in /// the meantime, but the function will have been applied only once to the stored value. @@ -2352,14 +2521,13 @@ impl AtomicPtr { /// ordering of this operation. The first describes the required ordering for /// when the operation finally succeeds while the second describes the /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicPtr::compare_exchange`] respectively. + /// orderings of [`AtomicBool::compare_exchange`] respectively. /// /// Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// **Note:** This method is only available on platforms that support atomic operations on `u8`. /// /// # Considerations /// @@ -2367,8 +2535,7 @@ impl AtomicPtr { /// critical section or mutex. /// /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], - /// which is a particularly common pitfall for pointers! + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap @@ -2377,27 +2544,24 @@ impl AtomicPtr { /// /// ```rust /// - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let ptr: *mut _ = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let new: *mut _ = &mut 10; - /// let result = some_ptr.update(Ordering::SeqCst, Ordering::SeqCst, |_| new); - /// assert_eq!(result, ptr); - /// assert_eq!(some_ptr.load(Ordering::SeqCst), new); + /// let x = AtomicBool::new(false); + /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), false); + /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), true); + /// assert_eq!(x.load(Ordering::SeqCst), false); /// ``` #[inline] #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub fn update( &self, set_order: Ordering, fetch_order: Ordering, - mut f: impl FnMut(*mut T) -> *mut T, - ) -> *mut T { + mut f: impl FnMut(bool) -> bool, + ) -> bool { let mut prev = self.load(fetch_order); loop { match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { @@ -2406,6 +2570,27 @@ impl AtomicPtr { } } } +} + +#[cfg(target_has_atomic_load_store = "ptr")] +impl AtomicPtr { + /// Creates a new `AtomicPtr` initialized with a null pointer. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_ptr_null)] + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let atomic_ptr = AtomicPtr::<()>::null(); + /// assert!(atomic_ptr.load(Ordering::Relaxed).is_null()); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "atomic_ptr_null", issue = "150733")] + pub const fn null() -> AtomicPtr { + AtomicPtr::new(crate::ptr::null_mut()) + } /// Offsets the pointer's address by adding `val` (in units of `T`), /// returning the previous pointer. @@ -2713,75 +2898,6 @@ impl AtomicPtr { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_xor(self.as_ptr(), val, order).cast() } } - - /// Returns a mutable pointer to the underlying pointer. - /// - /// Doing non-atomic reads and writes on the resulting pointer can be a data race. - /// This method is mostly useful for FFI, where the function signature may use - /// `*mut *mut T` instead of `&AtomicPtr`. - /// - /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the - /// atomic types work with interior mutability. All modifications of an atomic change the value - /// through a shared reference, and can do so safely as long as they use atomic operations. Any - /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the - /// requirements of the [memory model]. - /// - /// # Examples - /// - /// ```ignore (extern-declaration) - /// use std::sync::atomic::AtomicPtr; - /// - /// extern "C" { - /// fn my_atomic_op(arg: *mut *mut u32); - /// } - /// - /// let mut value = 17; - /// let atomic = AtomicPtr::new(&mut value); - /// - /// // SAFETY: Safe as long as `my_atomic_op` is atomic. - /// unsafe { - /// my_atomic_op(atomic.as_ptr()); - /// } - /// ``` - /// - /// [memory model]: self#memory-model-for-atomic-accesses - #[inline] - #[stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_never_returns_null_ptr] - pub const fn as_ptr(&self) -> *mut *mut T { - self.v.get().cast() - } -} - -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "atomic_bool_from", since = "1.24.0")] -#[rustc_const_unstable(feature = "const_convert", issue = "143773")] -const impl From for AtomicBool { - /// Converts a `bool` into an `AtomicBool`. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::AtomicBool; - /// let atomic_bool = AtomicBool::from(true); - /// assert_eq!(format!("{atomic_bool:?}"), "true") - /// ``` - #[inline] - fn from(b: bool) -> Self { - Self::new(b) - } -} - -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "atomic_from", since = "1.23.0")] -#[rustc_const_unstable(feature = "const_convert", issue = "143773")] -const impl From<*mut T> for AtomicPtr { - /// Converts a `*mut T` into an `AtomicPtr`. - #[inline] - fn from(p: *mut T) -> Self { - Self::new(p) - } } #[allow(unused_macros)] // This macro ends up being unused on some architectures. @@ -2793,6 +2909,69 @@ macro_rules! if_8_bit { #[cfg(target_has_atomic_load_store)] macro_rules! atomic_int { + ( + $s_int_type:literal, $int_type:ident, $atomic_type:ident + ) => { + /// An integer type which can be safely shared between threads. + /// + /// This type has the same + #[doc = if_8_bit!( + $int_type, + yes = ["size, alignment, and bit validity"], + no = ["size and bit validity"], + )] + /// as the underlying integer type, [` + #[doc = $s_int_type] + /// `]. + #[doc = if_8_bit! { + $int_type, + no = [ + "However, the alignment of this type is always equal to its ", + "size, even on targets where [`", $s_int_type, "`] has a ", + "lesser alignment." + ], + }] + /// For more about the differences between atomic types and + /// non-atomic types as well as information about the portability of + /// this type, please see the [module-level documentation]. + /// + /// **Note:** This type is only available on platforms that support + /// atomic loads and stores of [` + #[doc = $s_int_type] + /// `]. + /// + /// [module-level documentation]: crate::sync::atomic + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + pub type $atomic_type = Atomic<$int_type>; + + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + impl fmt::Debug for $atomic_type { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.load(Ordering::Relaxed), f) + } + } + }; +} + +#[cfg(target_has_atomic_load_store = "8")] +atomic_int! { "i8", i8, AtomicI8 } +#[cfg(target_has_atomic_load_store = "8")] +atomic_int! { "u8", u8, AtomicU8 } +#[cfg(target_has_atomic_load_store = "16")] +atomic_int! { "i16", i16, AtomicI16 } +#[cfg(target_has_atomic_load_store = "16")] +atomic_int! { "u16", u16, AtomicU16 } +#[cfg(target_has_atomic_load_store = "32")] +atomic_int! { "i32", i32, AtomicI32 } +#[cfg(target_has_atomic_load_store = "32")] +atomic_int! { "u32", u32, AtomicU32 } +#[cfg(target_has_atomic_load_store = "64")] +atomic_int! { "i64", i64, AtomicI64 } +#[cfg(target_has_atomic_load_store = "64")] +atomic_int! { "u64", u64, AtomicU64 } + +#[cfg(any(target_has_atomic_load_store = "128", doc))] +macro_rules! atomic_int_legacy { ($cfg_base:meta, $cfg_cas:meta, $cfg_align:meta, @@ -4018,160 +4197,8 @@ macro_rules! atomic_int { } } -#[cfg(target_has_atomic_load_store = "8")] -atomic_int! { - target_has_atomic_load_store = "8", - target_has_atomic = "8", - target_has_atomic_primitive_alignment = "8", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i8", - "", - atomic_min, atomic_max, - 1, - i8 AtomicI8 -} -#[cfg(target_has_atomic_load_store = "8")] -atomic_int! { - target_has_atomic_load_store = "8", - target_has_atomic = "8", - target_has_atomic_primitive_alignment = "8", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u8", - "", - atomic_umin, atomic_umax, - 1, - u8 AtomicU8 -} -#[cfg(target_has_atomic_load_store = "16")] -atomic_int! { - target_has_atomic_load_store = "16", - target_has_atomic = "16", - target_has_atomic_primitive_alignment = "16", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i16", - "", - atomic_min, atomic_max, - 2, - i16 AtomicI16 -} -#[cfg(target_has_atomic_load_store = "16")] -atomic_int! { - target_has_atomic_load_store = "16", - target_has_atomic = "16", - target_has_atomic_primitive_alignment = "16", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u16", - "", - atomic_umin, atomic_umax, - 2, - u16 AtomicU16 -} -#[cfg(target_has_atomic_load_store = "32")] -atomic_int! { - target_has_atomic_load_store = "32", - target_has_atomic = "32", - target_has_atomic_primitive_alignment = "32", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i32", - "", - atomic_min, atomic_max, - 4, - i32 AtomicI32 -} -#[cfg(target_has_atomic_load_store = "32")] -atomic_int! { - target_has_atomic_load_store = "32", - target_has_atomic = "32", - target_has_atomic_primitive_alignment = "32", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u32", - "", - atomic_umin, atomic_umax, - 4, - u32 AtomicU32 -} -#[cfg(target_has_atomic_load_store = "64")] -atomic_int! { - target_has_atomic_load_store = "64", - target_has_atomic = "64", - target_has_atomic_primitive_alignment = "64", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i64", - "", - atomic_min, atomic_max, - 8, - i64 AtomicI64 -} -#[cfg(target_has_atomic_load_store = "64")] -atomic_int! { - target_has_atomic_load_store = "64", - target_has_atomic = "64", - target_has_atomic_primitive_alignment = "64", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u64", - "", - atomic_umin, atomic_umax, - 8, - u64 AtomicU64 -} #[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { +atomic_int_legacy! { target_has_atomic_load_store = "128", target_has_atomic = "128", target_has_atomic_primitive_alignment = "128", @@ -4190,7 +4217,7 @@ atomic_int! { i128 AtomicI128 } #[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { +atomic_int_legacy! { target_has_atomic_load_store = "128", target_has_atomic = "128", target_has_atomic_primitive_alignment = "128", @@ -4213,43 +4240,9 @@ atomic_int! { macro_rules! atomic_int_ptr_sized { ( $($target_pointer_width:literal $align:literal)* ) => { $( #[cfg(target_pointer_width = $target_pointer_width)] - atomic_int! { - target_has_atomic_load_store = "ptr", - target_has_atomic = "ptr", - target_has_atomic_primitive_alignment = "ptr", - stable(feature = "rust1", since = "1.0.0"), - stable(feature = "extended_compare_and_swap", since = "1.10.0"), - stable(feature = "atomic_debug", since = "1.3.0"), - stable(feature = "atomic_access", since = "1.15.0"), - stable(feature = "atomic_from", since = "1.23.0"), - stable(feature = "atomic_nand", since = "1.27.0"), - rustc_const_stable(feature = "const_ptr_sized_atomics", since = "1.24.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "isize", - "", - atomic_min, atomic_max, - $align, - isize AtomicIsize - } + atomic_int! { "isize", isize, AtomicIsize } #[cfg(target_pointer_width = $target_pointer_width)] - atomic_int! { - target_has_atomic_load_store = "ptr", - target_has_atomic = "ptr", - target_has_atomic_primitive_alignment = "ptr", - stable(feature = "rust1", since = "1.0.0"), - stable(feature = "extended_compare_and_swap", since = "1.10.0"), - stable(feature = "atomic_debug", since = "1.3.0"), - stable(feature = "atomic_access", since = "1.15.0"), - stable(feature = "atomic_from", since = "1.23.0"), - stable(feature = "atomic_nand", since = "1.27.0"), - rustc_const_stable(feature = "const_ptr_sized_atomics", since = "1.24.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "usize", - "", - atomic_umin, atomic_umax, - $align, - usize AtomicUsize - } + atomic_int! { "usize", usize, AtomicUsize } /// An [`AtomicIsize`] initialized to `0`. #[cfg(target_pointer_width = $target_pointer_width)] diff --git a/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr b/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr index e718aac5329a4..d247ee01f0a34 100644 --- a/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr +++ b/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr @@ -6,12 +6,8 @@ LL | FOO.fetch_add(1, Ordering::Relaxed) | note: inside `Atomic::::fetch_add` --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL - ::: $SRC_DIR/core/src/sync/atomic.rs:LL:COL - | - = note: in this macro invocation note: inside `atomic::atomic_add::` --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL - = note: this error originates in the macro `atomic_int` which comes from the expansion of the macro `atomic_int_ptr_sized` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0080]: constant accesses mutable global memory --> $DIR/const_refers_to_static.rs:16:14 diff --git a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr index 4d17a6d4b5e9e..96486eb661d5e 100644 --- a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr +++ b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr @@ -1,26 +1,13 @@ -error[E0599]: no associated function or constant named `from_mut` found for struct `Atomic` in the current scope +error[E0599]: the associated function or constant `from_mut` exists for struct `Atomic`, but its trait bounds were not satisfied --> $DIR/atomic-from-mut-not-available.rs:25:36 | LL | core::sync::atomic::AtomicU64::from_mut(&mut 0u64); - | ^^^^^^^^ associated function or constant not found in `Atomic` + | ^^^^^^^^ associated function or constant cannot be called on `Atomic` due to unsatisfied trait bounds | -note: if you're trying to build a new `Atomic`, consider using `Atomic::::new` which returns `Atomic` +note: if you're trying to build a new `Atomic`, consider using `Atomic::::new` which returns `Atomic<_>` --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL - ::: $SRC_DIR/core/src/sync/atomic.rs:LL:COL - | - = note: in this macro invocation - = note: the associated function or constant was found for - - `Atomic<*mut T>` - - `Atomic` - - `Atomic` - - `Atomic` - and 6 more types - = note: this error originates in the macro `atomic_int` (in Nightly builds, run with -Z macro-backtrace for more info) -help: there is an associated function `from` with a similar name - | -LL - core::sync::atomic::AtomicU64::from_mut(&mut 0u64); -LL + core::sync::atomic::AtomicU64::from(&mut 0u64); - | + = note: the following trait bounds were not satisfied: + `u64: AtomicAlignedPrimitive` error: aborting due to 1 previous error diff --git a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs index 0d5e09e6e2883..9f9f7e398a00b 100644 --- a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs +++ b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs @@ -23,5 +23,5 @@ fn main() { core::sync::atomic::AtomicU64::from_mut(&mut 0u64); - //[alignment_mismatch]~^ ERROR no associated function or constant named `from_mut` found for struct `Atomic` + //[alignment_mismatch]~^ERROR: the associated function or constant `from_mut` exists for struct `Atomic`, but its trait bounds were not satisfied [E0599] }