From a89229a69e0243acb62a9bd0debd58a1d991a0cd Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 00:22:16 -0400 Subject: [PATCH 01/16] use Atomic to implement atomic functions/methods --- library/core/src/sync/atomic.rs | 3957 ++++++++++--------------------- 1 file changed, 1316 insertions(+), 2641 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 8b770528a1736..21bc5d9796d9e 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -246,7 +246,7 @@ use self::Ordering::*; use crate::cell::UnsafeCell; use crate::hint::spin_loop; -use crate::intrinsics::AtomicOrdering as AO; +use crate::intrinsics::{AtomicOrdering as AO, transmute_unchecked}; use crate::mem::transmute; use crate::{fmt, intrinsics}; @@ -276,14 +276,36 @@ mod private { /// A marker trait for primitive types which can be modified atomically. /// -/// This is an implementation detail for [Atomic]\ which may disappear or be replaced at any time. -// -// # Safety -// -// Types implementing this trait must be primitives that can be modified atomically. -// -// The associated `Self::Storage` type must have the same size, but may have fewer validity -// invariants or a higher alignment requirement than `Self`. +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives that can be modified atomically. +/// +/// The associated `Self::Storage` type must have the same size, but may have a higher alignment +/// requirement than `Self`. Transmuting `Self::Storage` to/from `Self` via `unchecked_transmute` +/// must be valid. This also implies that transmuting/casting a `Self::Storage` reference/pointer to +/// `Self` is allowed, however there is no requirement when it comes to transmuting/casting a +/// `Self` reference/pointer to `Self::Storage`. Transmuting a `Self` reference/pointer to +/// `Self::Storage` can only be done if the reference/pointer has the same alignment as +/// `Self::Storage`. +/// +/// The associated `Self::OpType` type must be an integer or pointer type that is the same size as +/// `Self`. This is the type used with the atomic_load and atomic_store functions. Transmuting +/// between `Self::OpType` and `Self` must be valid. Casting (and then dereferencing) a pointer +/// from one type to the other must also be valid. +/// +/// Types implementing this trait will have the following methods implemented for +/// [Atomic]\ automatically: +/// +/// - new +/// - from_ptr +/// - get_mut +/// - get_mut_slice +/// - into_inner +/// - load +/// - store #[unstable( feature = "atomic_internals", reason = "implementation detail which may disappear or be replaced at any time", @@ -292,78 +314,261 @@ mod private { pub impl(self) unsafe trait AtomicPrimitive: Sized + Copy { /// Temporary implementation detail. type Storage: Sized; + /// Temporary implementation detail. + type OpType: Sized + Copy; +} + +/// A market trait for primitive types which can be modified atomically, and don't have special +/// atomic emulation fallbacks. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives that can modified atomically, and have no +/// special handling or emulation fallback. Implementing this trait automatically implements the +/// following atomic methods for [Atomic]\: +/// +/// - swap +/// - compare_exchange +/// - compare_exchange_weak +/// - compare_and_swap +/// - fetch_update +/// - try_update +/// - update +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicCas: AtomicPrimitive {} + +/// A market trait for primitive types which have the same alignment as their atomic counter-parts. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives whose alignment is the same as their atomic +/// counter parts. Implementing this trait automatically implements the following methods for +/// [Atomic]\: +/// +/// - from_mut +/// - from_mut_slice +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicAlignedPrimitive: AtomicPrimitive {} + +/// A marker trait for atomic integer types. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitive integer types. Implementing this trait +/// automatically implements the following atomic methods for [Atomic]\: +/// +/// - fetch_add +/// - fetch_sub +/// - fetch_max +/// - fetch_min +/// +/// The associated constant Self::IS_SIGNED must correspond to whether the actual integer type this +/// trait is implemented for is signed (true) or not (false). +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicInteger: AtomicCas { + /// Whether the integer type is signed or not + const IS_SIGNED: bool; } -macro impl_atomic_primitive { +/// A marker trait for atomic types that support bitwise operations +/// +/// # Safety +/// +/// Types implementing this trait must be primitives which support atomic bitwise operations. +/// Implementing this trait automatically implements the following atomic methods for +/// [Atomic]\: +/// +/// - fetch_nand +/// - fetch_and +/// - fetch_or +/// - fetch xor +/// +/// # Note +/// +/// The bitwise and, or, and xor operation are defined for both atomic pointers and integers, but +/// are only automatically implemented for integers. This is because the existing documentation +/// for these operations on pointers would be trampled, and it currently provides very useful +/// information on their use in relation to the strict provenance API. +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicBitwise: AtomicCas {} + +macro_rules! impl_atomic_traits { ( @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, - $cfg:meta + size($size:literal), + load_store ) => { #[unstable( feature = "atomic_internals", reason = "implementation detail which may disappear or be replaced at any time", issue = "none" )] - #[cfg($cfg)] + #[cfg(any(target_has_atomic_load_store = $size, doc))] unsafe impl $(<$T>)? AtomicPrimitive for $Primitive { type Storage = private::$Storage<$Operand>; + type OpType = $Operand; + } + }; + + ( + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + cas + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicCas for $Primitive {} + }; + + ( + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + aligned + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic_primitive_alignment = $size, doc))] + unsafe impl $(<$T>)? AtomicAlignedPrimitive for $Primitive {} + }; + + ( + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + signed + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicInteger for $Primitive { + const IS_SIGNED: bool = true; } - }, + }; ( - [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, - size($size:literal) + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), + unsigned ) => { - impl_atomic_primitive!( - @impl [$($T)?] $Primitive as $Storage<$Operand>, - target_has_atomic_load_store = $size - ); - }, + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicInteger for $Primitive { + const IS_SIGNED: bool = false; + } + }; ( - [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, + @impl [$($T:ident)?] $Primitive:ty as $Storage:ident<$Operand:ty>, size($size:literal), - doc + bitwise + ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic = $size, doc))] + unsafe impl $(<$T>)? AtomicBitwise for $Primitive {} + }; + + ( + [$T:ident] $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), $($trait:ident),+ $(,)? + ) => { + $( + impl_atomic_traits!(@impl [$T] $Primitive as $Storage<$Operand>, size($size), $trait); + )+ + }; + + ( + $Primitive:ty as $Storage:ident<$Operand:ty>, + size($size:literal), $($trait:ident),+ $(,)? ) => { - impl_atomic_primitive!( - @impl [$($T)?] $Primitive as $Storage<$Operand>, - any(target_has_atomic_load_store = $size, doc) - ); - }, + $( + impl_atomic_traits!(@impl [] $Primitive as $Storage<$Operand>, size($size), $trait); + )+ + }; } -impl_atomic_primitive!([] bool as Align1, size("8")); -impl_atomic_primitive!([] i8 as Align1, size("8")); -impl_atomic_primitive!([] u8 as Align1, size("8")); -impl_atomic_primitive!([] i16 as Align2, size("16")); -impl_atomic_primitive!([] u16 as Align2, size("16")); -impl_atomic_primitive!([] i32 as Align4, size("32")); -impl_atomic_primitive!([] u32 as Align4, size("32")); -impl_atomic_primitive!([] i64 as Align8, size("64")); -impl_atomic_primitive!([] u64 as Align8, size("64")); -impl_atomic_primitive!([] i128 as Align16, size("128"), doc); -impl_atomic_primitive!([] u128 as Align16, size("128"), doc); +impl_atomic_traits!(bool as Align1, size("8"), load_store, aligned); + +impl_atomic_traits!(i8 as Align1, size("8"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u8 as Align1, size("8"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(i16 as Align2, size("16"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u16 as Align2, size("16"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(i32 as Align4, size("32"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u32 as Align4, size("32"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(i64 as Align8, size("64"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!(u64 as Align8, size("64"), load_store, cas, aligned, unsigned, bitwise); +impl_atomic_traits!(i128 as Align16, size("128"), load_store, cas, aligned, signed, bitwise); +impl_atomic_traits!( + u128 as Align16, size("128"), load_store, cas, aligned, unsigned, bitwise +); #[cfg(target_pointer_width = "16")] -impl_atomic_primitive!([] isize as Align2, size("ptr")); +impl_atomic_traits!(isize as Align2, size("ptr"), load_store, cas, aligned, signed, bitwise); #[cfg(target_pointer_width = "32")] -impl_atomic_primitive!([] isize as Align4, size("ptr")); +impl_atomic_traits!(isize as Align4, size("ptr"), load_store, cas, aligned, signed, bitwise); #[cfg(target_pointer_width = "64")] -impl_atomic_primitive!([] isize as Align8, size("ptr")); +impl_atomic_traits!(isize as Align8, size("ptr"), load_store, cas, aligned, signed, bitwise); #[cfg(target_pointer_width = "16")] -impl_atomic_primitive!([] usize as Align2, size("ptr")); +impl_atomic_traits!( + usize as Align2, size("ptr"), load_store, cas, aligned, unsigned, bitwise +); #[cfg(target_pointer_width = "32")] -impl_atomic_primitive!([] usize as Align4, size("ptr")); +impl_atomic_traits!( + usize as Align4, size("ptr"), load_store, cas, aligned, unsigned, bitwise +); #[cfg(target_pointer_width = "64")] -impl_atomic_primitive!([] usize as Align8, size("ptr")); +impl_atomic_traits!( + usize as Align8, size("ptr"), load_store, cas, aligned, unsigned, bitwise +); #[cfg(target_pointer_width = "16")] -impl_atomic_primitive!([T] *mut T as Align2<*mut T>, size("ptr")); +impl_atomic_traits!([T] *mut T as Align2<*mut T>, size("ptr"), load_store, cas, aligned); #[cfg(target_pointer_width = "32")] -impl_atomic_primitive!([T] *mut T as Align4<*mut T>, size("ptr")); +impl_atomic_traits!([T] *mut T as Align4<*mut T>, size("ptr"), load_store, cas, aligned); #[cfg(target_pointer_width = "64")] -impl_atomic_primitive!([T] *mut T as Align8<*mut T>, size("ptr")); +impl_atomic_traits!([T] *mut T as Align8<*mut T>, size("ptr"), load_store, cas, aligned); /// A memory location which can be safely modified from multiple threads. /// @@ -414,17 +619,7 @@ const EMULATE_ATOMIC_BOOL: bool = cfg!(any( /// loads and stores of `u8`. #[cfg(target_has_atomic_load_store = "8")] #[stable(feature = "rust1", since = "1.0.0")] -pub type AtomicBool = Atomic; - -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "rust1", since = "1.0.0")] -impl Default for AtomicBool { - /// Creates an `AtomicBool` initialized to `false`. - #[inline] - fn default() -> Self { - Self::new(false) - } -} +pub type AtomicBool = Atomic::; /// A raw pointer type which can be safely shared between threads. /// @@ -436,15 +631,6 @@ impl Default for AtomicBool { #[stable(feature = "rust1", since = "1.0.0")] pub type AtomicPtr = Atomic<*mut T>; -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "rust1", since = "1.0.0")] -impl Default for AtomicPtr { - /// Creates a null `AtomicPtr`. - fn default() -> AtomicPtr { - AtomicPtr::new(crate::ptr::null_mut()) - } -} - /// Atomic memory orderings /// /// Memory orderings specify the way atomic operations synchronize memory. @@ -526,57 +712,110 @@ pub enum Ordering { SeqCst, } -/// An [`AtomicBool`] initialized to `false`. -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "rust1", since = "1.0.0")] -#[deprecated( - since = "1.34.0", - note = "the `new` function is now preferred", - suggestion = "AtomicBool::new(false)" -)] -#[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] -pub const ATOMIC_BOOL_INIT: AtomicBool = AtomicBool::new(false); - -#[cfg(target_has_atomic_load_store = "8")] -impl AtomicBool { - /// Creates a new `AtomicBool`. +impl Atomic { + /// Creates a new `Atomic` with the given value. /// /// # Examples /// /// ``` - /// use std::sync::atomic::AtomicBool; + /// use std::sync::atomic::Atomic; /// - /// let atomic_true = AtomicBool::new(true); - /// let atomic_false = AtomicBool::new(false); + /// let atomic = Atomic:::new(true); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_atomic_new", since = "1.24.0")] + #[stable(feature = "integer_atomics_stable", since="1.34.0")] + #[rustc_const_stable(feature= "const_atomic_new", since="1.34.0")] #[must_use] - pub const fn new(v: bool) -> AtomicBool { + pub const fn new(v: T) -> Self { + Self { + // SAFETY: + // By the contract of AtomicPrimitive, it is guaranteed that transmuting between T and + // T::Storage is valid. + v: UnsafeCell::new(unsafe { transmute_unchecked(v) }) + } + } + + /// Consumes the atomic and returns the contained value. + /// + /// # Examples + /// + /// ``` + /// use std::sync::atomic::Atomic; + /// + /// let some_bool = Atomic::::new(true); + /// assert_eq!(some_bool.into_inner(), true); + /// ``` + #[inline] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0")] + pub const fn into_inner(self) -> T { // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(v) } + // * Consuming self means there are no references to self/the underlying data + // * The UnsafeCell can be transmuted into T::Storage which by the contract of + // AtomicPrimitive can in turn by transmuted into T + // * No operations on Atomic can produce a T::Storage that is an invalid T. + unsafe { + // We can't use UnsafeCell's into_inner despite it being "older" (present since 1.0.0) + // because it's const variant wasn't stabilized until 1.83.0 and the const variant of + // this function was stablizied earlier in 1.79.0 + transmute_unchecked(self) + } + } + + /// Returns a mutable pointer to the underlying `T`. + /// + /// Doing non-atomic reads and writes on the resulting value can be a data race. This is mostly + /// useful for FFI, where the function signature may use `*mut T` instead of `&Atomic`. + /// + /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the + /// atomic types work with interior mutability. All modifications of an atomic change the value + /// through a shared reference, and can do so safely as long as they use atomic operations. Any + /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the + /// requirements of the [memory model]. + /// + /// # Examples + /// + /// ```ignore (extern-declaration) + /// use std::sync::atomic::Atomic; + /// + /// extern "C" { + /// fn my_atomic_op(arg: *mut bool); + /// } + /// + /// let mut atomic = Atomic::::new(true); + /// unsafe { + /// my_atomic_op(atomic.as_ptr()); + /// } + /// ``` + /// + /// [memory model]: self#memory-model-for-atomic-accesses + #[inline] + #[stable(feature = "atomic_as_ptr", since = "1.70.0")] + #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] + #[rustc_never_returns_null_ptr] + #[rustc_should_not_be_called_on_const_items] + pub const fn as_ptr(&self) -> *mut T { + self.v.get().cast() } - /// Creates a new `AtomicBool` from a pointer. + /// Creates a new `Atomic` from a pointer. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{self, AtomicBool}; + /// use std::sync::atomic::{Atomic, Ordering}; /// /// // Get a pointer to an allocated value /// let ptr: *mut bool = Box::into_raw(Box::new(false)); /// - /// assert!(ptr.cast::().is_aligned()); + /// assert!(ptr.cast::>().is_aligned()); /// /// { /// // Create an atomic view of the allocated value - /// let atomic = unsafe { AtomicBool::from_ptr(ptr) }; + /// let atomic = unsafe { Atomic::::from_ptr(ptr) }; /// /// // Use `atomic` for atomic operations, possibly share it with other threads - /// atomic.store(true, atomic::Ordering::Relaxed); + /// atomic.store(true, Ordering::Relaxed); /// } /// /// // It's ok to non-atomically access the value behind `ptr`, @@ -589,8 +828,7 @@ impl AtomicBool { /// /// # Safety /// - /// * `ptr` must be aligned to `align_of::()` (note that this is always true, since - /// `align_of::() == 1`). + /// * `ptr` must be aligned to `align_of::>()`. /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`. /// * You must adhere to the [Memory model for atomic accesses]. In particular, it is not /// allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different @@ -601,79 +839,60 @@ impl AtomicBool { #[inline] #[stable(feature = "atomic_from_ptr", since = "1.75.0")] #[rustc_const_stable(feature = "const_atomic_from_ptr", since = "1.84.0")] - pub const unsafe fn from_ptr<'a>(ptr: *mut bool) -> &'a AtomicBool { + pub const unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Atomic { // SAFETY: guaranteed by the caller unsafe { &*ptr.cast() } } - /// Creates a new pointer to `AtomicBool` from a pointer. + /// Creates a new pointer to an atomic from a pointer. /// /// This is useful if you want to do volatile atomic accesses, and thus avoid creating /// a reference to the destination. #[inline] #[unstable(feature = "atomic_volatile", issue = "158947")] - pub const fn from_ptr_raw(ptr: *mut bool) -> *const AtomicBool { + pub const fn from_ptr_raw(ptr: *mut T) -> *const T { ptr.cast_const().cast() } - /// Returns a mutable reference to the underlying [`bool`]. + /// Returns a mutable reference to the underlying `T`. /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. + /// This is safe because the mutable reference guarantees that no other threads are concurrently + /// accessing the atomic data. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let mut some_bool = AtomicBool::new(true); + /// let mut some_bool = Atomic::::new(true); /// assert_eq!(*some_bool.get_mut(), true); /// *some_bool.get_mut() = false; /// assert_eq!(some_bool.load(Ordering::SeqCst), false); /// ``` #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut(&mut self) -> &mut bool { - // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *self.as_ptr() } - } - - /// Gets atomic access to a `&mut bool`. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let mut some_bool = true; - /// let a = AtomicBool::from_mut(&mut some_bool); - /// a.store(false, Ordering::Relaxed); - /// assert_eq!(some_bool, false); - /// ``` - #[inline] - #[cfg(target_has_atomic_primitive_alignment = "8")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut(v: &mut bool) -> &mut Self { - // SAFETY: the mutable reference guarantees unique ownership, and - // alignment of both `bool` and `Self` is 1. - unsafe { &mut *(v as *mut bool as *mut Self) } + pub const fn get_mut(&mut self) -> &mut T { + // SAFETY: + // * The mutable reference guarantees unique ownership. + // * The contract of AtomicPrimitive guarantees that transmuting &mut T::Storage to &mut T + // is valid. + unsafe { transmute(self.v.get_mut()) } } - /// Gets non-atomic access to a `&mut [AtomicBool]` slice. + /// Gets non-atomic access to a `&mut [Atomic]` slice. /// /// This is safe because the mutable reference guarantees that no other threads are /// concurrently accessing the atomic data. /// /// # Examples /// - /// ```ignore-wasm - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// ```rust,ignore-wasm + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let mut some_bools = [const { AtomicBool::new(false) }; 10]; + /// let mut some_bools = [const { Atomic::::new(false) }; 10]; /// - /// let view: &mut [bool] = AtomicBool::get_mut_slice(&mut some_bools); + /// let view: &mut [bool] = Atomic::::get_mut_slice(&mut some_bools); /// assert_eq!(view, [false; 10]); /// view[..5].copy_from_slice(&[true; 5]); /// @@ -690,62 +909,12 @@ impl AtomicBool { #[inline] #[stable(feature = "atomic_from_mut", since = "1.98.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut_slice(this: &mut [Self]) -> &mut [bool] { + pub const fn get_mut_slice(this: &mut [Self]) -> &mut [T] { // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *(this as *mut [Self] as *mut [bool]) } - } - - /// Gets atomic access to a `&mut [bool]` slice. - /// - /// # Examples - /// - /// ```rust,ignore-wasm - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let mut some_bools = [false; 10]; - /// let a = &*AtomicBool::from_mut_slice(&mut some_bools); - /// std::thread::scope(|s| { - /// for i in 0..a.len() { - /// s.spawn(move || a[i].store(true, Ordering::Relaxed)); - /// } - /// }); - /// assert_eq!(some_bools, [true; 10]); - /// ``` - #[inline] - #[cfg(target_has_atomic_primitive_alignment = "8")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut_slice(v: &mut [bool]) -> &mut [Self] { - // SAFETY: the mutable reference guarantees unique ownership, and - // alignment of both `bool` and `Self` is 1. - unsafe { &mut *(v as *mut [bool] as *mut [Self]) } - } - - /// Consumes the atomic and returns the contained value. - /// - /// This is safe because passing `self` by value guarantees that no other threads are - /// concurrently accessing the atomic data. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::AtomicBool; - /// - /// let some_bool = AtomicBool::new(true); - /// assert_eq!(some_bool.into_inner(), true); - /// ``` - #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0")] - pub const fn into_inner(self) -> bool { - // SAFETY: - // * `Atomic` is essentially a transparent wrapper around `T`. - // * all operations on `Atomic` ensure that `T::Storage` remains - // a valid `bool`. - unsafe { transmute(self) } + unsafe { &mut *(this as *mut [Self] as *mut [T]) } } - /// Loads a value from the bool. + /// Loads a value atomically. /// /// `load` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. @@ -757,25 +926,27 @@ impl AtomicBool { /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let some_bool = AtomicBool::new(true); + /// let some_bool = Atomic::::new(true); /// /// assert_eq!(some_bool.load(Ordering::Relaxed), true); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const fn load(&self, order: Ordering) -> bool { + pub const fn load(&self, order: Ordering) -> T { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. unsafe { - atomic_load::<_, /* VOLATILE */ false>(self.v.get().cast::(), order) != 0 + transmute_unchecked( + atomic_load::<_, /* VOLATILE */ false>(self.v.get().cast::(), order) + ) } } - /// Perform a volatile atomic load from the bool. + /// Perform a volatile atomic load. /// /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. @@ -798,20 +969,49 @@ impl AtomicBool { /// # Panics /// /// Panics if `order` is [`Release`] or [`AcqRel`]. - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> bool { - // SAFETY: follows from our own safety requirements. - unsafe { - atomic_load::<_, /* VOLATILE */ true>(self.cast::(), order) != 0 - } - } - - /// Stores a value into the bool. /// - /// `store` takes an [`Ordering`] argument which describes the memory ordering + /// # Examples + /// + /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory + /// access, we may receive a buffer in shared memory from that device as follows: + /// + /// ```rust,no_run + /// #![feature(atomic_volatile)] + /// use std::sync::atomic::{fence, Atomic, Ordering}; + /// use std::ptr; + /// + /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); + /// let atomic_ptr = Atomic::<*mut u8>::from_ptr_raw(MMIO_ADDR); + /// + /// // Spin until we see a non-zero value. + /// let buf = 'buf: loop { + /// let buf = unsafe { atomic_ptr.load_volatile(Ordering::Relaxed) }; + /// if !buf.is_null() { + /// break 'buf buf; + /// } + /// }; + /// // Synchronize with the store whose value we just read. + /// // Note: a standard acquire fence may not be sufficient to synchronize with DMA devices. + /// // Depending on your target, you may have to use inline assembly to emit a special fence. + /// fence(Ordering::Acquire); + /// + /// // Now process the data in `buf`. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> T { + // SAFETY: follows from our own safety requirements. + unsafe { + transmute_unchecked( + atomic_load::<_, /* VOLATILE */ true>(self.cast::(), order) + ) + } + } + + /// Stores a value atomically. + /// + /// `store` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. /// /// # Panics @@ -821,27 +1021,29 @@ impl AtomicBool { /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let some_bool = AtomicBool::new(true); + /// let some_bool = Atomic::::new(true); /// /// some_bool.store(false, Ordering::Relaxed); /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn store(&self, val: bool, order: Ordering) { + pub const fn store(&self, val: T, order: Ordering) { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. unsafe { - atomic_store::<_, /* VOLATILE */ false>(self.v.get().cast::(), val as u8, order); + atomic_store::<_, /* VOLATILE */ false>( + self.v.get().cast::(), transmute_unchecked::<_, T::OpType>(val), order + ); } } - /// Performs a volatile atomic store into the bool. + /// Performs a volatile atomic store. /// /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. @@ -862,54 +1064,73 @@ impl AtomicBool { /// # Panics /// /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// + /// # Examples + /// + /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory + /// access, we may submit a buffer in shared memory to that device as follows: + /// + /// ```rust,no_run + /// #![feature(atomic_volatile)] + /// use std::sync::atomic::{fence, Atomic, Ordering}; + /// use std::ptr; + /// + /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); + /// let atomic_ptr = Atomic::<*mut u8>::from_ptr_raw(MMIO_ADDR); + /// + /// // Prepare some data for the DMA device. + /// # fn get_dma_buffer() -> *mut u8 { panic!() } + /// let buf = get_dma_buffer(); + /// + /// // Ensure the other side can synchronize with the store we do below. + /// // Note: a standard release fence may not be sufficient to synchronize with DMA devices. + /// // Depending on your target, you may have to use inline assembly to emit a special fence. + /// fence(Ordering::Release); + /// + /// unsafe { atomic_ptr.store_volatile(buf, Ordering::Relaxed) }; + /// ``` #[inline] #[unstable(feature = "atomic_volatile", issue = "158947")] #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const unsafe fn store_volatile(self: *const Self, val: bool, order: Ordering) { + pub const unsafe fn store_volatile(self: *const Self, val: T, order: Ordering) { // SAFETY: follows from our own safety requirements. unsafe { - atomic_store::<_, /* VOLATILE */ true>(self.cast::().cast_mut(), val as u8, order); + atomic_store::<_, /* VOLATILE */ true>( + self.cast::().cast_mut(), transmute_unchecked::<_, T::OpType>(val), order + ); } } +} - /// Stores a value into the bool, returning the previous value. +impl Atomic { + /// Stores a value into the pointer, returning the previous value. /// /// `swap` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let some_bool = AtomicBool::new(true); + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// assert_eq!(some_bool.swap(false, Ordering::Relaxed), true); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let some_atomic = Atomic::::new(5); + /// let value = some_atomic.swap(10, Ordering::Relaxed); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn swap(&self, val: bool, order: Ordering) -> bool { - if EMULATE_ATOMIC_BOOL { - if val { self.fetch_or(true, order) } else { self.fetch_and(false, order) } - } else { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_swap(self.v.get().cast::(), val as u8, order) != 0 } - } + pub const fn swap(&self, v: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_swap(self.as_ptr(), v, order) } } - /// Stores a value into the [`bool`] if the current value is the same as the `current` value. + /// Stores a value into the pointer if the current value is the same as the `current` value. /// /// The return value is always the previous value. If it is equal to `current`, then the value /// was updated. @@ -920,8 +1141,6 @@ impl AtomicBool { /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it /// happens, and using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. /// /// # Migrating to `compare_exchange` and `compare_exchange_weak` /// @@ -949,34 +1168,26 @@ impl AtomicBool { /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let some_bool = AtomicBool::new(true); - /// - /// assert_eq!(some_bool.compare_and_swap(true, false, Ordering::Relaxed), true); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let some_atomic = Atomic::::new(5); /// - /// assert_eq!(some_bool.compare_and_swap(true, true, Ordering::Relaxed), false); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let value = some_atomic.compare_and_swap(5, 10, Ordering::Relaxed); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[deprecated( since = "1.50.0", note = "Use `compare_exchange` or `compare_exchange_weak` instead" )] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { - match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { - Ok(x) => x, - Err(x) => x, - } + pub fn compare_and_swap(&self, current: T, new: T, order: Ordering) -> T { + self.compare_exchange(current, new, order, strongest_failure_ordering(order)) + .unwrap_or_else(|x| x) } - /// Stores a value into the [`bool`] if the current value is the same as the `current` value. + /// Stores a value into the pointer if the current value is the same as the `current` value. /// /// The return value is a result indicating whether the new value was written and containing /// the previous value. On success this value is guaranteed to be equal to `current`. @@ -990,27 +1201,20 @@ impl AtomicBool { /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// operations on pointers. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{AtomicPtr, Ordering}; /// - /// let some_bool = AtomicBool::new(true); + /// let ptr = &mut 5; + /// let some_ptr = AtomicPtr::new(ptr); /// - /// assert_eq!(some_bool.compare_exchange(true, - /// false, - /// Ordering::Acquire, - /// Ordering::Relaxed), - /// Ok(true)); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let other_ptr = &mut 10; /// - /// assert_eq!(some_bool.compare_exchange(true, true, - /// Ordering::SeqCst, - /// Ordering::Acquire), - /// Err(false)); - /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// let value = some_ptr.compare_exchange(ptr, other_ptr, + /// Ordering::SeqCst, Ordering::Relaxed); /// ``` /// /// # Considerations @@ -1020,70 +1224,31 @@ impl AtomicBool { /// `compare_exchange` with the previous load *does not ensure* that other threads have not /// changed the value in the interim. This is usually important when the *equality* check in /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the - /// [ABA problem]. + /// does not necessarily imply identity. This is a particularly common case for pointers, as + /// a pointer holding the same address does not imply that the same object exists at that + /// address! In this case, `compare_exchange` can lead to the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[doc(alias = "compare_and_swap")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub const fn compare_exchange( &self, - current: bool, - new: bool, + current: T, + new: T, success: Ordering, failure: Ordering, - ) -> Result { - if EMULATE_ATOMIC_BOOL { - // Pick the strongest ordering from success and failure. - let order = match (success, failure) { - (SeqCst, _) => SeqCst, - (_, SeqCst) => SeqCst, - (AcqRel, _) => AcqRel, - (_, AcqRel) => { - panic!("there is no such thing as an acquire-release failure ordering") - } - (Release, Acquire) => AcqRel, - (Acquire, _) => Acquire, - (_, Acquire) => Acquire, - (Release, Relaxed) => Release, - (_, Release) => panic!("there is no such thing as a release failure ordering"), - (Relaxed, Relaxed) => Relaxed, - }; - let old = if current == new { - // This is a no-op, but we still need to perform the operation - // for memory ordering reasons. - self.fetch_or(false, order) - } else { - // This sets the value to the new one and returns the old one. - self.swap(new, order) - }; - if old == current { Ok(old) } else { Err(old) } - } else { - // SAFETY: data races are prevented by atomic intrinsics. - match unsafe { - atomic_compare_exchange( - self.v.get().cast::(), - current as u8, - new as u8, - success, - failure, - ) - } { - Ok(x) => Ok(x != 0), - Err(x) => Err(x != 0), - } - } + ) -> Result { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_compare_exchange(self.as_ptr(), current, new, success, failure) } } - /// Stores a value into the [`bool`] if the current value is the same as the `current` value. + /// Stores a value into the pointer if the current value is the same as the `current` value. /// - /// Unlike [`AtomicBool::compare_exchange`], this function is allowed to spuriously fail even when the + /// Unlike [`AtomicPtr::compare_exchange`], this function is allowed to spuriously fail even when the /// comparison succeeds, which can result in more efficient code on some platforms. The /// return value is a result indicating whether the new value was written and containing the /// previous value. @@ -1096,20 +1261,16 @@ impl AtomicBool { /// of this operation [`Relaxed`], and using [`Release`] makes the successful load /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let val = AtomicBool::new(false); + /// let some_atomic = Atomic::::new(5); /// - /// let new = true; - /// let mut old = val.load(Ordering::Relaxed); + /// let mut old = some_atomic.load(Ordering::Relaxed); /// loop { - /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { + /// match some_atomic.compare_exchange_weak(old, 10, Ordering::SeqCst, Ordering::Relaxed) { /// Ok(_) => break, /// Err(x) => old = x, /// } @@ -1123,1127 +1284,1089 @@ impl AtomicBool { /// `compare_exchange` with the previous load *does not ensure* that other threads have not /// changed the value in the interim. This is usually important when the *equality* check in /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the - /// [ABA problem]. + /// does not necessarily imply identity. This is a particularly common case for pointers, as + /// a pointer holding the same address does not imply that the same object exists at that + /// address! In this case, `compare_exchange` can lead to the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[doc(alias = "compare_and_swap")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub const fn compare_exchange_weak( &self, - current: bool, - new: bool, + current: T, + new: T, success: Ordering, failure: Ordering, - ) -> Result { - if EMULATE_ATOMIC_BOOL { - return self.compare_exchange(current, new, success, failure); - } + ) -> Result { + // SAFETY: This intrinsic is unsafe because it operates on a raw pointer + // but we know for sure that the pointer is valid (we just got it from + // an `UnsafeCell` that we have by reference) and the atomic operation + // itself allows us to safely mutate the `UnsafeCell` contents. + unsafe { atomic_compare_exchange_weak(self.as_ptr(), current, new, success, failure) } + } - // SAFETY: data races are prevented by atomic intrinsics. - match unsafe { - atomic_compare_exchange_weak( - self.v.get().cast::(), - current as u8, - new as u8, - success, - failure, - ) - } { - Ok(x) => Ok(x != 0), - Err(x) => Err(x != 0), - } + /// An alias for [`AtomicPtr::try_update`]. + #[inline] + #[stable(feature = "atomic_fetch_update", since = "1.53.0")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + #[deprecated( + since = "1.99.0", + note = "renamed to `try_update` for consistency", + suggestion = "try_update" + )] + pub fn fetch_update( + &self, + set_order: Ordering, + fetch_order: Ordering, + f: F, + ) -> Result + where + F: FnMut(T) -> Option, + { + self.try_update(set_order, fetch_order, f) } - /// Logical "and" with a boolean value. + /// Fetches the value, and applies a function to it that returns an optional + /// new value. Returns a `Result` of `Ok(previous_value)` if the function + /// returned `Some(_)`, else `Err(previous_value)`. /// - /// Performs a logical "and" operation on the current value and the argument `val`, and sets - /// the new value to the result. + /// See also: [`update`](`Atomic::update`). /// - /// Returns the previous value. + /// Note: This may call the function multiple times if the value has been + /// changed from other threads in the meantime, as long as the function + /// returns `Some(_)`, but the function will have been applied only once to + /// the stored value. /// - /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. + /// `try_update` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. The first describes the required ordering for + /// when the operation finally succeeds while the second describes the + /// required ordering for loads. These correspond to the success and failure + /// orderings of [`Atomic::compare_exchange`] respectively. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// Using [`Acquire`] as success ordering makes the store part of this + /// operation [`Relaxed`], and using [`Release`] makes the final successful + /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], + /// [`Acquire`] or [`Relaxed`]. /// - /// # Examples + /// # Considerations /// - /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// This method is not magic; it is not provided by the hardware, and does not act like a + /// critical section or mutex. /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], + /// which is a particularly common pitfall for pointers! /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_and(true, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// # Examples + /// + /// ```rust + /// use std::sync::atomic::{Atomic, Ordering}; + /// + /// let some_atomic = Atomic::::new(5); + /// + /// assert_eq!(some_atomic.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(5)); + /// let result = some_atomic.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| { + /// if x == 5 { + /// Some(10) + /// } else { + /// None + /// } + /// }); + /// assert_eq!(result, Ok(5)); + /// assert_eq!(some_atomic.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "atomic_try_update", since = "1.95.0")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_and(&self, val: bool, order: Ordering) -> bool { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_and(self.v.get().cast::(), val as u8, order) != 0 } + pub fn try_update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: impl FnMut(T) -> Option, + ) -> Result { + let mut prev = self.load(fetch_order); + while let Some(next) = f(prev) { + match self.compare_exchange_weak(prev, next, set_order, fetch_order) { + x @ Ok(_) => return x, + Err(next_prev) => prev = next_prev, + } + } + Err(prev) } - /// Logical "nand" with a boolean value. + /// Fetches the value, applies a function to it that it return a new value. + /// The new value is stored and the old value is returned. /// - /// Performs a logical "nand" operation on the current value and the argument `val`, and sets - /// the new value to the result. + /// See also: [`try_update`](`AtomicPtr::try_update`). /// - /// Returns the previous value. + /// Note: This may call the function multiple times if the value has been changed from other threads in + /// the meantime, but the function will have been applied only once to the stored value. /// - /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. + /// `update` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. The first describes the required ordering for + /// when the operation finally succeeds while the second describes the + /// required ordering for loads. These correspond to the success and failure + /// orderings of [`AtomicPtr::compare_exchange`] respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// operations on pointers. + /// + /// # Considerations + /// + /// This method is not magic; it is not provided by the hardware, and does not act like a + /// critical section or mutex. + /// + /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], + /// which is a particularly common pitfall for pointers! + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap /// /// # Examples /// - /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// ```rust /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_nand(true, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst) as usize, 0); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// let some_atomic = Atomic::new(5); /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// let result = some_atomic.update(Ordering::SeqCst, Ordering::SeqCst, |_| 10); + /// assert_eq!(result, ptr); + /// assert_eq!(some_atomic.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "atomic_try_update", since = "1.95.0")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_nand(&self, val: bool, order: Ordering) -> bool { - // We can't use atomic_nand here because it can result in a bool with - // an invalid value. This happens because the atomic operation is done - // with an 8-bit integer internally, which would set the upper 7 bits. - // So we just use fetch_xor or swap instead. - if val { - // !(x & true) == !x - // We must invert the bool. - self.fetch_xor(true, order) - } else { - // !(x & false) == true - // We must set the bool to true. - self.swap(true, order) + pub fn update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: impl FnMut(T) -> T, + ) -> T { + let mut prev = self.load(fetch_order); + loop { + match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { + Ok(x) => break x, + Err(next_prev) => prev = next_prev, + } } } +} - /// Logical "or" with a boolean value. +impl Atomic { + /// Gets atomic access to a `&mut T`. /// - /// Performs a logical "or" operation on the current value and the argument `val`, and sets the - /// new value to the result. + /// **Note:** This function is only available on targets where `Atomic` has the same + /// alignment as `T`. /// - /// Returns the previous value. + /// # Examples /// - /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. + /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// let mut some_bool = true; + /// let a = Atomic::::from_mut(&mut some_bool); + /// a.store(false, Ordering::Relaxed); + /// assert_eq!(some_bool, false); + /// ``` + #[inline] + #[cfg(target_has_atomic_primitive_alignment = "8")] + #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut(v: &mut T) -> &mut Self { + const { + assert!( + align_of::() == align_of::(), + "This function is only available if Atomic and T have the same alignment" + ); + } + + // SAFETY: + // * The mutable reference guarantees unique ownership + // * The contract of AtomicPrimitive requires that T::Storage has the same, or higher + // alignment as T, casting from a higher alignment is safe. + // * The contract of AtomicPrimitive requires that T::Storage has the same size as T + // * No atomic operations are capable of transforming the underlying data into an invalid + // variant of T. + unsafe { &mut *(v as *mut T as *mut Self) } + } + + /// Gets atomic access to a `&mut [T]` slice. + /// + /// **Note:** This function is only available on targets where `Atomic` has the same + /// alignment as `T` /// /// # Examples /// + /// ```rust,ignore-wasm + /// use std::sync::atomic::{Atomic, Ordering}; + /// + /// let mut some_bools = [false; 10]; + /// let a = &*Atomic::::from_mut_slice(&mut some_bools); + /// std::thread::scope(|s| { + /// for i in 0..a.len() { + /// s.spawn(move || a[i].store(true, Ordering::Relaxed)); + /// } + /// }); + /// assert_eq!(some_bools, [true; 10]); /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + #[inline] + #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut_slice(v: &mut [T]) -> &mut [Self] { + const { + assert!( + align_of::() == align_of::(), + "This function is only available if Atomic and T have the same alignment" + ); + } + + // SAFETY: + // * The mutable reference guarantees unique ownership + // * The alignment of both T and Self is the same. + // * Since the alignment and size of Self and T are the same, and they have the same + // invariants, they can be freely cast between one another. + unsafe { &mut *(v as *mut [T] as *mut [Self]) } + } +} + +impl Atomic { + /// Adds to the current value, returning the previous value. /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// This operation wraps around on overflow. /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_or(true, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// `fetch_add` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// # Examples + /// + /// ``` + /// use std::sync::atomic{Atomic, Ordering}; + /// + /// let foo = Atomic::::new(0); + /// assert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_or(&self, val: bool, order: Ordering) -> bool { + pub const fn fetch_add(&self, val: T, order: Ordering) -> T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_or(self.v.get().cast::(), val as u8, order) != 0 } + unsafe { atomic_add(self.as_ptr(), val, order) } } - /// Logical "xor" with a boolean value. - /// - /// Performs a logical "xor" operation on the current value and the argument `val`, and sets - /// the new value to the result. + /// Subtracts from the current value, returning the previous value. /// - /// Returns the previous value. + /// This operation wraps around on overflow. /// - /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// `fetch_sub` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; - /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), true); - /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_xor(true, Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// use std::sync::atomic{Atomic, Ordering}; /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// let foo = Atomic::::new(20); + /// assert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); /// ``` #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_xor(&self, val: bool, order: Ordering) -> bool { + pub const fn fetch_sub(&self, val: T, order: Ordering) -> T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_xor(self.v.get().cast::(), val as u8, order) != 0 } + unsafe { atomic_sub(self.as_ptr(), val, order) } } - /// Logical "not" with a boolean value. + /// Maximum with the current value. /// - /// Performs a logical "not" operation on the current value, and sets - /// the new value to the result. + /// Finds the maximum of the current value and the argument `val`, and + /// sets the new value to the result. /// /// Returns the previous value. /// - /// `fetch_not` takes an [`Ordering`] argument which describes the memory ordering + /// `fetch_max` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. - /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let foo = AtomicBool::new(true); - /// assert_eq!(foo.fetch_not(Ordering::SeqCst), true); - /// assert_eq!(foo.load(Ordering::SeqCst), false); + /// let foo = Atomic::::new(23); + /// assert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23); + /// assert_eq!(foo.load(Ordering::SeqCst), 42); + /// ``` /// - /// let foo = AtomicBool::new(false); - /// assert_eq!(foo.fetch_not(Ordering::SeqCst), false); - /// assert_eq!(foo.load(Ordering::SeqCst), true); + /// If you want to obtain the maximum value in one step, you can use the following: + /// + /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; + /// + /// let foo = Atomic::::new(23); + /// let bar = 42; + /// let max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar); + /// assert!(max_foo == 42); /// ``` #[inline] - #[stable(feature = "atomic_bool_fetch_not", since = "1.81.0")] + #[stable(feature = "atomic_min_max", since = "1.45.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_not(&self, order: Ordering) -> bool { - self.fetch_xor(true, order) + pub const fn fetch_max(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + if T::IS_SIGNED { + unsafe { atomic_max(self.as_ptr(), val, order) } + } + else { + unsafe { atomic_umax(self.as_ptr(), val, order) } + } } - /// Returns a mutable pointer to the underlying [`bool`]. + /// Minimum with the current value. /// - /// Doing non-atomic reads and writes on the resulting boolean can be a data race. - /// This method is mostly useful for FFI, where the function signature may use - /// `*mut bool` instead of `&AtomicBool`. + /// Finds the minimum of the current value and the argument `val`, and + /// sets the new value to the result. /// - /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the - /// atomic types work with interior mutability. All modifications of an atomic change the value - /// through a shared reference, and can do so safely as long as they use atomic operations. Any - /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the - /// requirements of the [memory model]. + /// Returns the previous value. + /// + /// `fetch_min` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// # Examples /// - /// ```ignore (extern-declaration) - /// # fn main() { - /// use std::sync::atomic::AtomicBool; + /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// extern "C" { - /// fn my_atomic_op(arg: *mut bool); - /// } + /// let foo = Atomic::::new(23); + /// assert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 23); + /// assert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 22); + /// ``` + /// + /// If you want to obtain the minimum value in one step, you can use the following: /// - /// let mut atomic = AtomicBool::new(true); - /// unsafe { - /// my_atomic_op(atomic.as_ptr()); - /// } - /// # } /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// [memory model]: self#memory-model-for-atomic-accesses - #[inline] - #[stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_never_returns_null_ptr] - #[rustc_should_not_be_called_on_const_items] - pub const fn as_ptr(&self) -> *mut bool { - self.v.get().cast() - } - - /// An alias for [`AtomicBool::try_update`]. + /// let foo = Atomic::::new(23); + /// let bar = 12; + /// let min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar); + /// assert_eq!(min_foo, 12); + /// ``` #[inline] - #[stable(feature = "atomic_fetch_update", since = "1.53.0")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - #[deprecated( - since = "1.99.0", - note = "renamed to `try_update` for consistency", - suggestion = "try_update" - )] - pub fn fetch_update( - &self, - set_order: Ordering, - fetch_order: Ordering, - f: F, - ) -> Result - where - F: FnMut(bool) -> Option, - { - self.try_update(set_order, fetch_order, f) + pub const fn fetch_min(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + if ::IS_SIGNED { + unsafe { atomic_min(self.as_ptr(), val, order) } + } + else { + unsafe { atomic_umin(self.as_ptr(), val, order) } + } } +} - /// Fetches the value, and applies a function to it that returns an optional - /// new value. Returns a `Result` of `Ok(previous_value)` if the function - /// returned `Some(_)`, else `Err(previous_value)`. +impl Atomic { + /// Bitwise "nand" with the current value. /// - /// See also: [`update`](`AtomicBool::update`). + /// Performs a bitwise "nand" operation on the current value and the argument `val`, and + /// sets the new value to the result. /// - /// Note: This may call the function multiple times if the value has been - /// changed from other threads in the meantime, as long as the function - /// returns `Some(_)`, but the function will have been applied only once to - /// the stored value. + /// Returns the previous value. /// - /// `try_update` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering for - /// when the operation finally succeeds while the second describes the - /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicBool::compare_exchange`] respectively. + /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// Using [`Acquire`] as success ordering makes the store part of this - /// operation [`Relaxed`], and using [`Release`] makes the final successful - /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], - /// [`Acquire`] or [`Relaxed`]. + /// # Examples /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on `u8`. + /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// # Considerations + /// let foo = Atomic::::new(0x13); + /// assert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13); + /// assert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31)); + /// ``` + #[inline] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_nand(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_nand(self.as_ptr(), val, order) } + } + + /// Bitwise "and" with the current value. /// - /// This method is not magic; it is not provided by the hardware, and does not act like a - /// critical section or mutex. + /// Performs a bitwise "and" operation on the current value and the argument `val`, and + /// sets the new value to the result. /// - /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. + /// Returns the previous value. /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// # Examples /// - /// ```rust - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let x = AtomicBool::new(false); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(false)); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(false)); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(true)); - /// assert_eq!(x.load(Ordering::SeqCst), false); + /// let foo = Atomic::::new(0b101101); + /// assert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b100001); /// ``` #[inline] - #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn try_update( - &self, - set_order: Ordering, - fetch_order: Ordering, - mut f: impl FnMut(bool) -> Option, - ) -> Result { - let mut prev = self.load(fetch_order); - while let Some(next) = f(prev) { - match self.compare_exchange_weak(prev, next, set_order, fetch_order) { - x @ Ok(_) => return x, - Err(next_prev) => prev = next_prev, - } - } - Err(prev) + pub const fn fetch_and(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_and(self.as_ptr(), val, order) } } - /// Fetches the value, applies a function to it that it return a new value. - /// The new value is stored and the old value is returned. + /// Bitwise "or" with the current value. /// - /// See also: [`try_update`](`AtomicBool::try_update`). + /// Performs a bitwise "or" operation on the current value and the argument `val`, and + /// sets the new value to the result. /// - /// Note: This may call the function multiple times if the value has been changed from other threads in - /// the meantime, but the function will have been applied only once to the stored value. + /// Returns the previous value. /// - /// `update` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. The first describes the required ordering for - /// when the operation finally succeeds while the second describes the - /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicBool::compare_exchange`] respectively. + /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load - /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// # Examples /// - /// **Note:** This method is only available on platforms that support atomic operations on `u8`. + /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// # Considerations + /// let foo = Atomic::::new(0b101101); + /// assert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b111111); + /// ``` + #[inline] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_or(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_or(self.as_ptr(), val, order) } + } + + /// Bitwise "xor" with the current value. /// - /// This method is not magic; it is not provided by the hardware, and does not act like a - /// critical section or mutex. + /// Performs a bitwise "xor" operation on the current value and the argument `val`, and + /// sets the new value to the result. /// - /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. + /// Returns the previous value. /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// # Examples /// - /// ```rust - /// - /// use std::sync::atomic::{AtomicBool, Ordering}; + /// ``` + /// use std::sync::atomic::{Atomic, Ordering}; /// - /// let x = AtomicBool::new(false); - /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), false); - /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), true); - /// assert_eq!(x.load(Ordering::SeqCst), false); + /// let foo = Atomic::::new(0b101101); + /// assert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b011110); /// ``` #[inline] - #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "8")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn update( - &self, - set_order: Ordering, - fetch_order: Ordering, - mut f: impl FnMut(bool) -> bool, - ) -> bool { - let mut prev = self.load(fetch_order); - loop { - match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { - Ok(x) => break x, - Err(next_prev) => prev = next_prev, - } - } + pub const fn fetch_xor(&self, val: T, order: Ordering) -> T { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_xor(self.as_ptr(), val, order) } } } -#[cfg(target_has_atomic_load_store = "ptr")] -impl AtomicPtr { - /// Creates a new `AtomicPtr`. +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +impl Default for Atomic { + #[inline] + fn default() -> Self { + Self::new(T::default()) + } +} + +#[stable(feature = "integer_atomics_stable", since = "1.34.0")] +#[rustc_const_unstable(feature = "const_convert", issue = "143773")] +impl From for Atomic { + fn from(value: T) -> Self { + Self::new(value) + } +} + +/// An [`AtomicBool`] initialized to `false`. +#[cfg(target_has_atomic_load_store = "8")] +#[stable(feature = "rust1", since = "1.0.0")] +#[deprecated( + since = "1.34.0", + note = "the `new` function is now preferred", + suggestion = "AtomicBool::new(false)" +)] +#[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] +pub const ATOMIC_BOOL_INIT: AtomicBool = AtomicBool::new(false); + +#[cfg(target_has_atomic_load_store = "8")] +impl AtomicBool { + /// Stores a value into the bool, returning the previous value. + /// + /// `swap` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::AtomicPtr; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let atomic_ptr = AtomicPtr::new(ptr); + /// let some_bool = AtomicBool::new(true); + /// + /// assert_eq!(some_bool.swap(false, Ordering::Relaxed), true); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_atomic_new", since = "1.24.0")] - pub const fn new(p: *mut T) -> AtomicPtr { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(p) } + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn swap(&self, val: bool, order: Ordering) -> bool { + if EMULATE_ATOMIC_BOOL { + if val { self.fetch_or(true, order) } else { self.fetch_and(false, order) } + } else { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_swap(self.v.get().cast::(), val as u8, order) != 0 } + } } - /// Creates a new `AtomicPtr` from a pointer. + /// Stores a value into the [`bool`] if the current value is the same as the `current` value. /// - /// # Examples + /// The return value is always the previous value. If it is equal to `current`, then the value + /// was updated. /// - /// ``` - /// use std::sync::atomic::{self, AtomicPtr}; + /// `compare_and_swap` also takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. Notice that even when using [`AcqRel`], the operation + /// might fail and hence just perform an `Acquire` load, but not have `Release` semantics. + /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it + /// happens, and using [`Release`] makes the load part [`Relaxed`]. /// - /// // Get a pointer to an allocated value - /// let ptr: *mut *mut u8 = Box::into_raw(Box::new(std::ptr::null_mut())); + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// - /// assert!(ptr.cast::>().is_aligned()); + /// # Migrating to `compare_exchange` and `compare_exchange_weak` /// - /// { - /// // Create an atomic view of the allocated value - /// let atomic = unsafe { AtomicPtr::from_ptr(ptr) }; + /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for + /// memory orderings: /// - /// // Use `atomic` for atomic operations, possibly share it with other threads - /// atomic.store(std::ptr::NonNull::dangling().as_ptr(), atomic::Ordering::Relaxed); - /// } - /// - /// // It's ok to non-atomically access the value behind `ptr`, - /// // since the reference to the atomic ended its lifetime in the block above - /// assert!(!unsafe { *ptr }.is_null()); - /// - /// // Deallocate the value - /// unsafe { drop(Box::from_raw(ptr)) } - /// ``` - /// - /// # Safety - /// - /// * `ptr` must be aligned to `align_of::>()` (note that on some platforms this - /// can be bigger than `align_of::<*mut T>()`). - /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`. - /// * You must adhere to the [Memory model for atomic accesses]. In particular, it is not - /// allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different - /// sizes, without synchronization. - /// - /// [valid]: crate::ptr#safety - /// [Memory model for atomic accesses]: self#memory-model-for-atomic-accesses - #[inline] - #[stable(feature = "atomic_from_ptr", since = "1.75.0")] - #[rustc_const_stable(feature = "const_atomic_from_ptr", since = "1.84.0")] - pub const unsafe fn from_ptr<'a>(ptr: *mut *mut T) -> &'a AtomicPtr { - // SAFETY: guaranteed by the caller - unsafe { &*ptr.cast() } - } - - /// Creates a new pointer to `AtomicPtr` from a pointer. - /// - /// This is useful if you want to do volatile atomic accesses, and thus avoid creating - /// a reference to the destination. - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - pub const fn from_ptr_raw(ptr: *mut *mut T) -> *const AtomicPtr { - ptr.cast_const().cast() - } - - /// Creates a new `AtomicPtr` initialized with a null pointer. - /// - /// # Examples - /// - /// ``` - /// #![feature(atomic_ptr_null)] - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst /// - /// let atomic_ptr = AtomicPtr::<()>::null(); - /// assert!(atomic_ptr.load(Ordering::Relaxed).is_null()); - /// ``` - #[inline] - #[must_use] - #[unstable(feature = "atomic_ptr_null", issue = "150733")] - pub const fn null() -> AtomicPtr { - AtomicPtr::new(crate::ptr::null_mut()) - } - - /// Returns a mutable reference to the underlying pointer. + /// `compare_and_swap` and `compare_exchange` also differ in their return type. You can use + /// `compare_exchange(...).unwrap_or_else(|x| x)` to recover the behavior of `compare_and_swap`, + /// but in most cases it is more idiomatic to check whether the return value is `Ok` or `Err` + /// rather than to infer success vs failure based on the value that was read. /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. + /// During migration, consider whether it makes sense to use `compare_exchange_weak` instead. + /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, + /// which allows the compiler to generate better assembly code when the compare and swap + /// is used in a loop. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let mut data = 10; - /// let mut atomic_ptr = AtomicPtr::new(&mut data); - /// let mut other_data = 5; - /// *atomic_ptr.get_mut() = &mut other_data; - /// assert_eq!(unsafe { *atomic_ptr.load(Ordering::SeqCst) }, 5); - /// ``` - #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut(&mut self) -> &mut *mut T { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { &mut *self.as_ptr() } - } - - /// Gets atomic access to a pointer. - /// - /// **Note:** This function is only available on targets where `AtomicPtr` has the same alignment as `*const T` + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// # Examples + /// let some_bool = AtomicBool::new(true); /// - /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// assert_eq!(some_bool.compare_and_swap(true, false, Ordering::Relaxed), true); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// - /// let mut data = 123; - /// let mut some_ptr = &mut data as *mut i32; - /// let a = AtomicPtr::from_mut(&mut some_ptr); - /// let mut other_data = 456; - /// a.store(&mut other_data, Ordering::Relaxed); - /// assert_eq!(unsafe { *some_ptr }, 456); + /// assert_eq!(some_bool.compare_and_swap(true, true, Ordering::Relaxed), false); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` #[inline] - #[cfg(target_has_atomic_primitive_alignment = "ptr")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut(v: &mut *mut T) -> &mut Self { - let [] = [(); align_of::>() - align_of::<*mut ()>()]; - // SAFETY: - // - the mutable reference guarantees unique ownership. - // - the alignment of `*mut T` and `Self` is the same on all platforms - // supported by rust, as verified above. - unsafe { &mut *(v as *mut *mut T as *mut Self) } + #[deprecated( + since = "1.50.0", + note = "Use `compare_exchange` or `compare_exchange_weak` instead" + )] + #[cfg(target_has_atomic = "8")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { + match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { + Ok(x) => x, + Err(x) => x, + } } - /// Gets non-atomic access to a `&mut [AtomicPtr]` slice. - /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. - /// - /// # Examples - /// - /// ```ignore-wasm - /// use std::ptr::null_mut; - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let mut some_ptrs = [const { AtomicPtr::new(null_mut::()) }; 10]; + /// Stores a value into the [`bool`] if the current value is the same as the `current` value. /// - /// let view: &mut [*mut String] = AtomicPtr::get_mut_slice(&mut some_ptrs); - /// assert_eq!(view, [null_mut::(); 10]); - /// view - /// .iter_mut() - /// .enumerate() - /// .for_each(|(i, ptr)| *ptr = Box::into_raw(Box::new(format!("iteration#{i}")))); + /// The return value is a result indicating whether the new value was written and containing + /// the previous value. On success this value is guaranteed to be equal to `current`. /// - /// std::thread::scope(|s| { - /// for ptr in &some_ptrs { - /// s.spawn(move || { - /// let ptr = ptr.load(Ordering::Relaxed); - /// assert!(!ptr.is_null()); - /// - /// let name = unsafe { Box::from_raw(ptr) }; - /// println!("Hello, {name}!"); - /// }); - /// } - /// }); - /// ``` - #[inline] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut_slice(this: &mut [Self]) -> &mut [*mut T] { - // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *(this as *mut [Self] as *mut [*mut T]) } - } - - /// Gets atomic access to a slice of pointers. + /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// **Note:** This function is only available on targets where `AtomicPtr` has the same alignment as `*const T` + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// - /// ```ignore-wasm - /// use std::ptr::null_mut; - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let mut some_ptrs = [null_mut::(); 10]; - /// let a = &*AtomicPtr::from_mut_slice(&mut some_ptrs); - /// std::thread::scope(|s| { - /// for i in 0..a.len() { - /// s.spawn(move || { - /// let name = Box::new(format!("thread{i}")); - /// a[i].store(Box::into_raw(name), Ordering::Relaxed); - /// }); - /// } - /// }); - /// for p in some_ptrs { - /// assert!(!p.is_null()); - /// let name = unsafe { Box::from_raw(p) }; - /// println!("Hello, {name}!"); - /// } /// ``` - #[inline] - #[cfg(target_has_atomic_primitive_alignment = "ptr")] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut_slice(v: &mut [*mut T]) -> &mut [Self] { - // SAFETY: - // - the mutable reference guarantees unique ownership. - // - the alignment of `*mut T` and `Self` is the same on all platforms - // supported by rust, as verified above. - unsafe { &mut *(v as *mut [*mut T] as *mut [Self]) } - } - - /// Consumes the atomic and returns the contained value. + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// This is safe because passing `self` by value guarantees that no other threads are - /// concurrently accessing the atomic data. + /// let some_bool = AtomicBool::new(true); /// - /// # Examples + /// assert_eq!(some_bool.compare_exchange(true, + /// false, + /// Ordering::Acquire, + /// Ordering::Relaxed), + /// Ok(true)); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// + /// assert_eq!(some_bool.compare_exchange(true, true, + /// Ordering::SeqCst, + /// Ordering::Acquire), + /// Err(false)); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); /// ``` - /// use std::sync::atomic::AtomicPtr; /// - /// let mut data = 5; - /// let atomic_ptr = AtomicPtr::new(&mut data); - /// assert_eq!(unsafe { *atomic_ptr.into_inner() }, 5); - /// ``` - #[inline] - #[stable(feature = "atomic_access", since = "1.15.0")] - #[rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0")] - pub const fn into_inner(self) -> *mut T { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(self) } - } - - /// Loads a value from the pointer. - /// - /// `load` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. - /// - /// # Panics - /// - /// Panics if `order` is [`Release`] or [`AcqRel`]. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// # Considerations /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides + /// of CAS operations. In particular, a load of the value followed by a successful + /// `compare_exchange` with the previous load *does not ensure* that other threads have not + /// changed the value in the interim. This is usually important when the *equality* check in + /// the `compare_exchange` is being used to check the *identity* of a value, but equality + /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the + /// [ABA problem]. /// - /// let value = some_ptr.load(Ordering::Relaxed); - /// ``` + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[doc(alias = "compare_and_swap")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const fn load(&self, order: Ordering) -> *mut T { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { - atomic_load::<_, /* VOLATILE */ false>(self.as_ptr(), order) + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_exchange( + &self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering, + ) -> Result { + if EMULATE_ATOMIC_BOOL { + // Pick the strongest ordering from success and failure. + let order = match (success, failure) { + (SeqCst, _) => SeqCst, + (_, SeqCst) => SeqCst, + (AcqRel, _) => AcqRel, + (_, AcqRel) => { + panic!("there is no such thing as an acquire-release failure ordering") + } + (Release, Acquire) => AcqRel, + (Acquire, _) => Acquire, + (_, Acquire) => Acquire, + (Release, Relaxed) => Release, + (_, Release) => panic!("there is no such thing as a release failure ordering"), + (Relaxed, Relaxed) => Relaxed, + }; + let old = if current == new { + // This is a no-op, but we still need to perform the operation + // for memory ordering reasons. + self.fetch_or(false, order) + } else { + // This sets the value to the new one and returns the old one. + self.swap(new, order) + }; + if old == current { Ok(old) } else { Err(old) } + } else { + // SAFETY: data races are prevented by atomic intrinsics. + match unsafe { + atomic_compare_exchange( + self.v.get().cast::(), + current as u8, + new as u8, + success, + failure, + ) + } { + Ok(x) => Ok(x != 0), + Err(x) => Err(x != 0), + } } } - /// Perform a volatile atomic load from the pointer. - /// - /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. - /// - #[doc = include_str!("./atomic_load_volatile.md")] - /// - /// # Safety - /// - /// Behavior is undefined if any of the following conditions are violated: - /// - /// * `self` must be [valid] for reads, or `self` must point to memory - /// outside of all Rust allocations and reading from that memory must: - /// - not trap, and - /// - not cause any memory inside a Rust allocation to be modified. - /// - /// * `self` must be aligned to `align_of::>()` (note that on some platforms this - /// can be bigger than `align_of::<*mut T>()`). - /// - /// * Reading from `self` must produce a properly initialized value of type `*mut T`. + /// Stores a value into the [`bool`] if the current value is the same as the `current` value. /// - /// [valid]: core::ptr#safety + /// Unlike [`AtomicBool::compare_exchange`], this function is allowed to spuriously fail even when the + /// comparison succeeds, which can result in more efficient code on some platforms. The + /// return value is a result indicating whether the new value was written and containing the + /// previous value. /// - /// # Panics + /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// Panics if `order` is [`Release`] or [`AcqRel`]. + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// - /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory - /// access, we may receive a buffer in shared memory from that device as follows: - /// - /// ```rust,no_run - /// #![feature(atomic_volatile)] - /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; - /// use std::ptr; - /// - /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); - /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); - /// - /// // Spin until we see a non-zero value. - /// let buf = 'buf: loop { - /// let buf = unsafe { atomic_ptr.load_volatile(Ordering::Relaxed) }; - /// if !buf.is_null() { - /// break 'buf buf; - /// } - /// }; - /// // Synchronize with the store whose value we just read. - /// // Note: a standard acquire fence may not be sufficient to synchronize with DMA devices. - /// // Depending on your target, you may have to use inline assembly to emit a special fence. - /// fence(Ordering::Acquire); - /// - /// // Now process the data in `buf`. /// ``` - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> *mut T { - // SAFETY: follows from our own safety requirements. - unsafe { - atomic_load::<_, /* VOLATILE */ true>(self.cast::<*mut T>(), order) - } - } - - /// Stores a value into the pointer. - /// - /// `store` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. - /// - /// # Panics - /// - /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// # Examples + /// let val = AtomicBool::new(false); /// + /// let new = true; + /// let mut old = val.load(Ordering::Relaxed); + /// loop { + /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { + /// Ok(_) => break, + /// Err(x) => old = x, + /// } + /// } /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// # Considerations /// - /// let other_ptr = &mut 10; + /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides + /// of CAS operations. In particular, a load of the value followed by a successful + /// `compare_exchange` with the previous load *does not ensure* that other threads have not + /// changed the value in the interim. This is usually important when the *equality* check in + /// the `compare_exchange` is being used to check the *identity* of a value, but equality + /// does not necessarily imply identity. In this case, `compare_exchange` can lead to the + /// [ABA problem]. /// - /// some_ptr.store(other_ptr, Ordering::Relaxed); - /// ``` + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "rust1", since = "1.0.0")] + #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[doc(alias = "compare_and_swap")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn store(&self, ptr: *mut T, order: Ordering) { + pub const fn compare_exchange_weak( + &self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering, + ) -> Result { + if EMULATE_ATOMIC_BOOL { + return self.compare_exchange(current, new, success, failure); + } + // SAFETY: data races are prevented by atomic intrinsics. - unsafe { - atomic_store::<_, /* VOLATILE */ false>(self.as_ptr(), ptr, order); + match unsafe { + atomic_compare_exchange_weak( + self.v.get().cast::(), + current as u8, + new as u8, + success, + failure, + ) + } { + Ok(x) => Ok(x != 0), + Err(x) => Err(x != 0), } } - /// Performs a volatile atomic store into the pointer. - /// - /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. - /// - #[doc = include_str!("./atomic_store_volatile.md")] - /// - /// # Safety - /// - /// Behavior is undefined if any of the following conditions are violated: - /// - /// * `self` must be either [valid] for writes, or `self` must point to memory - /// outside of all Rust allocations and writing to that memory must: - /// - not trap, and - /// - not cause any memory inside a Rust allocation to be modified. + /// Logical "and" with a boolean value. /// - /// * `self` must be aligned to `align_of::>()` (note that on some platforms this - /// can be bigger than `align_of::<*mut T>()`). + /// Performs a logical "and" operation on the current value and the argument `val`, and sets + /// the new value to the result. /// - /// [valid]: core::ptr#safety + /// Returns the previous value. /// - /// # Panics + /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// - /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory - /// access, we may submit a buffer in shared memory to that device as follows: - /// - /// ```rust,no_run - /// #![feature(atomic_volatile)] - /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; - /// use std::ptr; - /// - /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); - /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); + /// ``` + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// // Prepare some data for the DMA device. - /// # fn get_dma_buffer() -> *mut u8 { panic!() } - /// let buf = get_dma_buffer(); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// // Ensure the other side can synchronize with the store we do below. - /// // Note: a standard release fence may not be sufficient to synchronize with DMA devices. - /// // Depending on your target, you may have to use inline assembly to emit a special fence. - /// fence(Ordering::Release); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_and(true, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// unsafe { atomic_ptr.store_volatile(buf, Ordering::Relaxed) }; + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// ``` #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const unsafe fn store_volatile(self: *const Self, ptr: *mut T, order: Ordering) { - // SAFETY: follows from our own safety requirements. - unsafe { - atomic_store::<_, /* VOLATILE */ true>(self.cast::<*mut T>().cast_mut(), ptr, order); - } + pub const fn fetch_and(&self, val: bool, order: Ordering) -> bool { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_and(self.v.get().cast::(), val as u8, order) != 0 } } - /// Stores a value into the pointer, returning the previous value. + /// Logical "nand" with a boolean value. /// - /// `swap` takes an [`Ordering`] argument which describes the memory ordering + /// Performs a logical "nand" operation on the current value and the argument `val`, and sets + /// the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering /// of this operation. All ordering modes are possible. Note that using /// [`Acquire`] makes the store part of this operation [`Relaxed`], and /// using [`Release`] makes the load part [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let other_ptr = &mut 10; + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_nand(true, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst) as usize, 0); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// let value = some_ptr.swap(other_ptr, Ordering::Relaxed); + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub const fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_swap(self.as_ptr(), ptr, order) } + pub const fn fetch_nand(&self, val: bool, order: Ordering) -> bool { + // We can't use atomic_nand here because it can result in a bool with + // an invalid value. This happens because the atomic operation is done + // with an 8-bit integer internally, which would set the upper 7 bits. + // So we just use fetch_xor or swap instead. + if val { + // !(x & true) == !x + // We must invert the bool. + self.fetch_xor(true, order) + } else { + // !(x & false) == true + // We must set the bool to true. + self.swap(true, order) + } } - /// Stores a value into the pointer if the current value is the same as the `current` value. - /// - /// The return value is always the previous value. If it is equal to `current`, then the value - /// was updated. - /// - /// `compare_and_swap` also takes an [`Ordering`] argument which describes the memory - /// ordering of this operation. Notice that even when using [`AcqRel`], the operation - /// might fail and hence just perform an `Acquire` load, but not have `Release` semantics. - /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it - /// happens, and using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. - /// - /// # Migrating to `compare_exchange` and `compare_exchange_weak` + /// Logical "or" with a boolean value. /// - /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for - /// memory orderings: + /// Performs a logical "or" operation on the current value and the argument `val`, and sets the + /// new value to the result. /// - /// Original | Success | Failure - /// -------- | ------- | ------- - /// Relaxed | Relaxed | Relaxed - /// Acquire | Acquire | Acquire - /// Release | Release | Relaxed - /// AcqRel | AcqRel | Acquire - /// SeqCst | SeqCst | SeqCst + /// Returns the previous value. /// - /// `compare_and_swap` and `compare_exchange` also differ in their return type. You can use - /// `compare_exchange(...).unwrap_or_else(|x| x)` to recover the behavior of `compare_and_swap`, - /// but in most cases it is more idiomatic to check whether the return value is `Ok` or `Err` - /// rather than to infer success vs failure based on the value that was read. + /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// - /// During migration, consider whether it makes sense to use `compare_exchange_weak` instead. - /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, - /// which allows the compiler to generate better assembly code when the compare and swap - /// is used in a loop. + /// **Note:** This method is only available on platforms that support atomic + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let other_ptr = &mut 10; + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_or(true, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let value = some_ptr.compare_and_swap(ptr, other_ptr, Ordering::Relaxed); + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] - #[deprecated( - since = "1.50.0", - note = "Use `compare_exchange` or `compare_exchange_weak` instead" - )] - #[cfg(target_has_atomic = "ptr")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_and_swap(&self, current: *mut T, new: *mut T, order: Ordering) -> *mut T { - match self.compare_exchange(current, new, order, strongest_failure_ordering(order)) { - Ok(x) => x, - Err(x) => x, - } + pub const fn fetch_or(&self, val: bool, order: Ordering) -> bool { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_or(self.v.get().cast::(), val as u8, order) != 0 } } - /// Stores a value into the pointer if the current value is the same as the `current` value. + /// Logical "xor" with a boolean value. /// - /// The return value is a result indicating whether the new value was written and containing - /// the previous value. On success this value is guaranteed to be equal to `current`. + /// Performs a logical "xor" operation on the current value and the argument `val`, and sets + /// the new value to the result. /// - /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. `success` describes the required ordering for the - /// read-modify-write operation that takes place if the comparison with `current` succeeds. - /// `failure` describes the required ordering for the load operation that takes place when - /// the comparison fails. Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// Returns the previous value. + /// + /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let ptr = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// - /// let other_ptr = &mut 10; + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_xor(true, Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// let value = some_ptr.compare_exchange(ptr, other_ptr, - /// Ordering::SeqCst, Ordering::Relaxed); + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// ``` - /// - /// # Considerations - /// - /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides - /// of CAS operations. In particular, a load of the value followed by a successful - /// `compare_exchange` with the previous load *does not ensure* that other threads have not - /// changed the value in the interim. This is usually important when the *equality* check in - /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. This is a particularly common case for pointers, as - /// a pointer holding the same address does not imply that the same object exists at that - /// address! In this case, `compare_exchange` can lead to the [ABA problem]. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] - #[cfg(target_has_atomic = "ptr")] + #[stable(feature = "rust1", since = "1.0.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange( - &self, - current: *mut T, - new: *mut T, - success: Ordering, - failure: Ordering, - ) -> Result<*mut T, *mut T> { + pub const fn fetch_xor(&self, val: bool, order: Ordering) -> bool { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_compare_exchange(self.as_ptr(), current, new, success, failure) } + unsafe { atomic_xor(self.v.get().cast::(), val as u8, order) != 0 } } - /// Stores a value into the pointer if the current value is the same as the `current` value. - /// - /// Unlike [`AtomicPtr::compare_exchange`], this function is allowed to spuriously fail even when the - /// comparison succeeds, which can result in more efficient code on some platforms. The - /// return value is a result indicating whether the new value was written and containing the - /// previous value. + /// Logical "not" with a boolean value. /// - /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. `success` describes the required ordering for the - /// read-modify-write operation that takes place if the comparison with `current` succeeds. - /// `failure` describes the required ordering for the load operation that takes place when - /// the comparison fails. Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// Performs a logical "not" operation on the current value, and sets + /// the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_not` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Examples /// /// ``` - /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let some_ptr = AtomicPtr::new(&mut 5); + /// let foo = AtomicBool::new(true); + /// assert_eq!(foo.fetch_not(Ordering::SeqCst), true); + /// assert_eq!(foo.load(Ordering::SeqCst), false); /// - /// let new = &mut 10; - /// let mut old = some_ptr.load(Ordering::Relaxed); - /// loop { - /// match some_ptr.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { - /// Ok(_) => break, - /// Err(x) => old = x, - /// } - /// } + /// let foo = AtomicBool::new(false); + /// assert_eq!(foo.fetch_not(Ordering::SeqCst), false); + /// assert_eq!(foo.load(Ordering::SeqCst), true); /// ``` - /// - /// # Considerations - /// - /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides - /// of CAS operations. In particular, a load of the value followed by a successful - /// `compare_exchange` with the previous load *does not ensure* that other threads have not - /// changed the value in the interim. This is usually important when the *equality* check in - /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. This is a particularly common case for pointers, as - /// a pointer holding the same address does not imply that the same object exists at that - /// address! In this case, `compare_exchange` can lead to the [ABA problem]. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap #[inline] - #[stable(feature = "extended_compare_and_swap", since = "1.10.0")] - #[cfg(target_has_atomic = "ptr")] + #[stable(feature = "atomic_bool_fetch_not", since = "1.81.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] - pub fn compare_exchange_weak( - &self, - current: *mut T, - new: *mut T, - success: Ordering, - failure: Ordering, - ) -> Result<*mut T, *mut T> { - // SAFETY: This intrinsic is unsafe because it operates on a raw pointer - // but we know for sure that the pointer is valid (we just got it from - // an `UnsafeCell` that we have by reference) and the atomic operation - // itself allows us to safely mutate the `UnsafeCell` contents. - unsafe { atomic_compare_exchange_weak(self.as_ptr(), current, new, success, failure) } + pub const fn fetch_not(&self, order: Ordering) -> bool { + self.fetch_xor(true, order) } - /// An alias for [`AtomicPtr::try_update`]. + /// An alias for [`AtomicBool::try_update`]. #[inline] #[stable(feature = "atomic_fetch_update", since = "1.53.0")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] #[deprecated( @@ -2256,17 +2379,18 @@ impl AtomicPtr { set_order: Ordering, fetch_order: Ordering, f: F, - ) -> Result<*mut T, *mut T> + ) -> Result where - F: FnMut(*mut T) -> Option<*mut T>, + F: FnMut(bool) -> Option, { self.try_update(set_order, fetch_order, f) } + /// Fetches the value, and applies a function to it that returns an optional /// new value. Returns a `Result` of `Ok(previous_value)` if the function /// returned `Some(_)`, else `Err(previous_value)`. /// - /// See also: [`update`](`AtomicPtr::update`). + /// See also: [`update`](`AtomicBool::update`). /// /// Note: This may call the function multiple times if the value has been /// changed from other threads in the meantime, as long as the function @@ -2277,7 +2401,7 @@ impl AtomicPtr { /// ordering of this operation. The first describes the required ordering for /// when the operation finally succeeds while the second describes the /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicPtr::compare_exchange`] respectively. + /// orderings of [`AtomicBool::compare_exchange`] respectively. /// /// Using [`Acquire`] as success ordering makes the store part of this /// operation [`Relaxed`], and using [`Release`] makes the final successful @@ -2285,7 +2409,7 @@ impl AtomicPtr { /// [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// operations on `u8`. /// /// # Considerations /// @@ -2293,8 +2417,7 @@ impl AtomicPtr { /// critical section or mutex. /// /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], - /// which is a particularly common pitfall for pointers! + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap @@ -2302,34 +2425,25 @@ impl AtomicPtr { /// # Examples /// /// ```rust - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let ptr: *mut _ = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let new: *mut _ = &mut 10; - /// assert_eq!(some_ptr.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(ptr)); - /// let result = some_ptr.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| { - /// if x == ptr { - /// Some(new) - /// } else { - /// None - /// } - /// }); - /// assert_eq!(result, Ok(ptr)); - /// assert_eq!(some_ptr.load(Ordering::SeqCst), new); + /// let x = AtomicBool::new(false); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(false)); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(false)); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(true)); + /// assert_eq!(x.load(Ordering::SeqCst), false); /// ``` #[inline] #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub fn try_update( &self, set_order: Ordering, fetch_order: Ordering, - mut f: impl FnMut(*mut T) -> Option<*mut T>, - ) -> Result<*mut T, *mut T> { + mut f: impl FnMut(bool) -> Option, + ) -> Result { let mut prev = self.load(fetch_order); while let Some(next) = f(prev) { match self.compare_exchange_weak(prev, next, set_order, fetch_order) { @@ -2343,7 +2457,7 @@ impl AtomicPtr { /// Fetches the value, applies a function to it that it return a new value. /// The new value is stored and the old value is returned. /// - /// See also: [`try_update`](`AtomicPtr::try_update`). + /// See also: [`try_update`](`AtomicBool::try_update`). /// /// Note: This may call the function multiple times if the value has been changed from other threads in /// the meantime, but the function will have been applied only once to the stored value. @@ -2352,14 +2466,13 @@ impl AtomicPtr { /// ordering of this operation. The first describes the required ordering for /// when the operation finally succeeds while the second describes the /// required ordering for loads. These correspond to the success and failure - /// orderings of [`AtomicPtr::compare_exchange`] respectively. + /// orderings of [`AtomicBool::compare_exchange`] respectively. /// /// Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// - /// **Note:** This method is only available on platforms that support atomic - /// operations on pointers. + /// **Note:** This method is only available on platforms that support atomic operations on `u8`. /// /// # Considerations /// @@ -2367,8 +2480,7 @@ impl AtomicPtr { /// critical section or mutex. /// /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem], - /// which is a particularly common pitfall for pointers! + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem]. /// /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap @@ -2377,27 +2489,24 @@ impl AtomicPtr { /// /// ```rust /// - /// use std::sync::atomic::{AtomicPtr, Ordering}; - /// - /// let ptr: *mut _ = &mut 5; - /// let some_ptr = AtomicPtr::new(ptr); + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let new: *mut _ = &mut 10; - /// let result = some_ptr.update(Ordering::SeqCst, Ordering::SeqCst, |_| new); - /// assert_eq!(result, ptr); - /// assert_eq!(some_ptr.load(Ordering::SeqCst), new); + /// let x = AtomicBool::new(false); + /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), false); + /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| !x), true); + /// assert_eq!(x.load(Ordering::SeqCst), false); /// ``` #[inline] #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(target_has_atomic = "ptr")] + #[cfg(target_has_atomic = "8")] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub fn update( &self, set_order: Ordering, fetch_order: Ordering, - mut f: impl FnMut(*mut T) -> *mut T, - ) -> *mut T { + mut f: impl FnMut(bool) -> bool, + ) -> bool { let mut prev = self.load(fetch_order); loop { match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { @@ -2406,6 +2515,27 @@ impl AtomicPtr { } } } +} + +#[cfg(target_has_atomic_load_store = "ptr")] +impl AtomicPtr { + /// Creates a new `AtomicPtr` initialized with a null pointer. + /// + /// # Examples + /// + /// ``` + /// #![feature(atomic_ptr_null)] + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let atomic_ptr = AtomicPtr::<()>::null(); + /// assert!(atomic_ptr.load(Ordering::Relaxed).is_null()); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "atomic_ptr_null", issue = "150733")] + pub const fn null() -> AtomicPtr { + AtomicPtr::new(crate::ptr::null_mut()) + } /// Offsets the pointer's address by adding `val` (in units of `T`), /// returning the previous pointer. @@ -2713,75 +2843,6 @@ impl AtomicPtr { // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_xor(self.as_ptr(), val, order).cast() } } - - /// Returns a mutable pointer to the underlying pointer. - /// - /// Doing non-atomic reads and writes on the resulting pointer can be a data race. - /// This method is mostly useful for FFI, where the function signature may use - /// `*mut *mut T` instead of `&AtomicPtr`. - /// - /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the - /// atomic types work with interior mutability. All modifications of an atomic change the value - /// through a shared reference, and can do so safely as long as they use atomic operations. Any - /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the - /// requirements of the [memory model]. - /// - /// # Examples - /// - /// ```ignore (extern-declaration) - /// use std::sync::atomic::AtomicPtr; - /// - /// extern "C" { - /// fn my_atomic_op(arg: *mut *mut u32); - /// } - /// - /// let mut value = 17; - /// let atomic = AtomicPtr::new(&mut value); - /// - /// // SAFETY: Safe as long as `my_atomic_op` is atomic. - /// unsafe { - /// my_atomic_op(atomic.as_ptr()); - /// } - /// ``` - /// - /// [memory model]: self#memory-model-for-atomic-accesses - #[inline] - #[stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_never_returns_null_ptr] - pub const fn as_ptr(&self) -> *mut *mut T { - self.v.get().cast() - } -} - -#[cfg(target_has_atomic_load_store = "8")] -#[stable(feature = "atomic_bool_from", since = "1.24.0")] -#[rustc_const_unstable(feature = "const_convert", issue = "143773")] -const impl From for AtomicBool { - /// Converts a `bool` into an `AtomicBool`. - /// - /// # Examples - /// - /// ``` - /// use std::sync::atomic::AtomicBool; - /// let atomic_bool = AtomicBool::from(true); - /// assert_eq!(format!("{atomic_bool:?}"), "true") - /// ``` - #[inline] - fn from(b: bool) -> Self { - Self::new(b) - } -} - -#[cfg(target_has_atomic_load_store = "ptr")] -#[stable(feature = "atomic_from", since = "1.23.0")] -#[rustc_const_unstable(feature = "const_convert", issue = "143773")] -const impl From<*mut T> for AtomicPtr { - /// Converts a `*mut T` into an `AtomicPtr`. - #[inline] - fn from(p: *mut T) -> Self { - Self::new(p) - } } #[allow(unused_macros)] // This macro ends up being unused on some architectures. @@ -2793,22 +2854,9 @@ macro_rules! if_8_bit { #[cfg(target_has_atomic_load_store)] macro_rules! atomic_int { - ($cfg_base:meta, - $cfg_cas:meta, - $cfg_align:meta, - $stable:meta, - $stable_cxchg:meta, - $stable_debug:meta, - $stable_access:meta, - $stable_from:meta, - $stable_nand:meta, - $const_stable_new:meta, - $const_stable_into_inner:meta, - $s_int_type:literal, - $extra_feature:expr, - $min_fn:ident, $max_fn:ident, - $align:expr, - $int_type:ident $atomic_type:ident) => { + ( + $s_int_type:literal, $int_type:ident, $atomic_type:ident + ) => { /// An integer type which can be safely shared between threads. /// /// This type has the same @@ -2839,1417 +2887,46 @@ macro_rules! atomic_int { /// `]. /// /// [module-level documentation]: crate::sync::atomic - #[$stable] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] pub type $atomic_type = Atomic<$int_type>; - #[$stable] - impl Default for $atomic_type { - #[inline] - fn default() -> Self { - Self::new(Default::default()) - } - } - - #[$stable_from] - #[rustc_const_unstable(feature = "const_convert", issue = "143773")] - const impl From<$int_type> for $atomic_type { - #[doc = concat!("Converts an `", stringify!($int_type), "` into an `", stringify!($atomic_type), "`.")] - #[inline] - fn from(v: $int_type) -> Self { Self::new(v) } - } - - #[$stable_debug] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] impl fmt::Debug for $atomic_type { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.load(Ordering::Relaxed), f) } } - - impl $atomic_type { - /// Creates a new atomic integer. - /// - /// # Examples - /// - #[cfg_attr($cfg_base, doc = "```")] - #[cfg_attr(not($cfg_base), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] - /// - #[doc = concat!("let atomic_forty_two = ", stringify!($atomic_type), "::new(42);")] - /// ``` - #[inline] - #[$stable] - #[$const_stable_new] - #[must_use] - pub const fn new(v: $int_type) -> Self { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(v) } - } - - /// Creates a new reference to an atomic integer from a pointer. - /// - /// # Examples - /// - #[cfg_attr($cfg_base, doc = "```rust")] - #[cfg_attr(not($cfg_base), doc = "```rust,compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{self, ", stringify!($atomic_type), "};")] - /// - /// // Get a pointer to an allocated value - #[doc = concat!("let ptr: *mut ", stringify!($int_type), " = Box::into_raw(Box::new(0));")] - /// - #[doc = concat!("assert!(ptr.cast::<", stringify!($atomic_type), ">().is_aligned());")] - /// - /// { - /// // Create an atomic view of the allocated value - // SAFETY: this is a doc comment, tidy, it can't hurt you (also guaranteed by the construction of `ptr` and the assert above) - #[doc = concat!(" let atomic = unsafe {", stringify!($atomic_type), "::from_ptr(ptr) };")] - /// - /// // Use `atomic` for atomic operations, possibly share it with other threads - /// atomic.store(1, atomic::Ordering::Relaxed); - /// } - /// - /// // It's ok to non-atomically access the value behind `ptr`, - /// // since the reference to the atomic ended its lifetime in the block above - /// assert_eq!(unsafe { *ptr }, 1); - /// - /// // Deallocate the value - /// unsafe { drop(Box::from_raw(ptr)) } - /// ``` - /// - /// # Safety - /// - /// * `ptr` must be aligned to - #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] - #[doc = if_8_bit!{ - $int_type, - yes = [ - " (note that this is always true, since `align_of::<", - stringify!($atomic_type), ">() == 1`)." - ], - no = [ - " (note that on some platforms this can be bigger than `align_of::<", - stringify!($int_type), ">()`)." - ], - }] - /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`. - /// * You must adhere to the [Memory model for atomic accesses]. In particular, it is not - /// allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different - /// sizes, without synchronization. - /// - /// [valid]: crate::ptr#safety - /// [Memory model for atomic accesses]: self#memory-model-for-atomic-accesses - #[inline] - #[stable(feature = "atomic_from_ptr", since = "1.75.0")] - #[rustc_const_stable(feature = "const_atomic_from_ptr", since = "1.84.0")] - pub const unsafe fn from_ptr<'a>(ptr: *mut $int_type) -> &'a $atomic_type { - // SAFETY: guaranteed by the caller - unsafe { &*ptr.cast() } - } - - /// Creates a new pointer to an atomic integer from a pointer. - /// - /// This is useful if you want to do volatile atomic accesses, and thus avoid creating - /// a reference to the destination. - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - pub const fn from_ptr_raw(ptr: *mut $int_type) -> *const $atomic_type { - ptr.cast_const().cast() - } - - /// Returns a mutable reference to the underlying integer. - /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. - /// - /// # Examples - /// - #[cfg_attr($cfg_base, doc = "```")] - #[cfg_attr(not($cfg_base), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let mut some_var = ", stringify!($atomic_type), "::new(10);")] - /// assert_eq!(*some_var.get_mut(), 10); - /// *some_var.get_mut() = 5; - /// assert_eq!(some_var.load(Ordering::SeqCst), 5); - /// ``` - #[inline] - #[$stable_access] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut(&mut self) -> &mut $int_type { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { &mut *self.as_ptr() } - } - - #[doc = concat!("Get atomic access to a `&mut ", stringify!($int_type), "`.")] - /// - #[doc = if_8_bit! { - $int_type, - no = [ - "**Note:** This function is only available on targets where `", - stringify!($atomic_type), "` has the same alignment as `", stringify!($int_type), "`." - ], - }] - /// - /// # Examples - /// - #[cfg_attr($cfg_align, doc = "```rust")] - #[cfg_attr(not($cfg_align), doc = "```rust,compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - /// let mut some_int = 123; - #[doc = concat!("let a = ", stringify!($atomic_type), "::from_mut(&mut some_int);")] - /// a.store(100, Ordering::Relaxed); - /// assert_eq!(some_int, 100); - /// ``` - /// - #[inline] - #[cfg(any($cfg_align, doc))] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut(v: &mut $int_type) -> &mut Self { - let [] = [(); align_of::() - align_of::<$int_type>()]; - // SAFETY: - // - the mutable reference guarantees unique ownership. - // - the alignment of `$int_type` and `Self` is the - // same, as promised by $cfg_align and verified above. - unsafe { &mut *(v as *mut $int_type as *mut Self) } - } - - #[doc = concat!("Get non-atomic access to a `&mut [", stringify!($atomic_type), "]` slice")] - /// - /// This is safe because the mutable reference guarantees that no other threads are - /// concurrently accessing the atomic data. - /// - /// # Examples - /// - #[cfg_attr($cfg_base, doc = "```ignore-wasm")] - #[cfg_attr(not($cfg_base), doc = "```ignore-wasm,compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let mut some_ints = [const { ", stringify!($atomic_type), "::new(0) }; 10];")] - /// - #[doc = concat!("let view: &mut [", stringify!($int_type), "] = ", stringify!($atomic_type), "::get_mut_slice(&mut some_ints);")] - /// assert_eq!(view, [0; 10]); - /// view - /// .iter_mut() - /// .enumerate() - /// .for_each(|(idx, int)| *int = idx as _); - /// - /// std::thread::scope(|s| { - /// some_ints - /// .iter() - /// .enumerate() - /// .for_each(|(idx, int)| { - /// s.spawn(move || assert_eq!(int.load(Ordering::Relaxed), idx as _)); - /// }) - /// }); - /// ``` - #[inline] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn get_mut_slice(this: &mut [Self]) -> &mut [$int_type] { - // SAFETY: the mutable reference guarantees unique ownership. - unsafe { &mut *(this as *mut [Self] as *mut [$int_type]) } - } - - #[doc = concat!("Get atomic access to a `&mut [", stringify!($int_type), "]` slice.")] - /// - #[doc = if_8_bit! { - $int_type, - no = [ - "**Note:** This function is only available on targets where `", - stringify!($atomic_type), "` has the same alignment as `", stringify!($int_type), "`." - ], - }] - /// - /// # Examples - /// - #[cfg_attr($cfg_align, doc = "```ignore-wasm")] - #[cfg_attr(not($cfg_align), doc = "```ignore-wasm,compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - /// let mut some_ints = [0; 10]; - #[doc = concat!("let a = &*", stringify!($atomic_type), "::from_mut_slice(&mut some_ints);")] - /// std::thread::scope(|s| { - /// for i in 0..a.len() { - /// s.spawn(move || a[i].store(i as _, Ordering::Relaxed)); - /// } - /// }); - /// for (i, n) in some_ints.into_iter().enumerate() { - /// assert_eq!(i, n as usize); - /// } - /// ``` - #[inline] - #[cfg(any($cfg_align, doc))] - #[stable(feature = "atomic_from_mut", since = "1.98.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - pub const fn from_mut_slice(v: &mut [$int_type]) -> &mut [Self] { - let [] = [(); align_of::() - align_of::<$int_type>()]; - // SAFETY: - // - the mutable reference guarantees unique ownership. - // - the alignment of `$int_type` and `Self` is the - // same, as promised by $cfg_align and verified above. - unsafe { &mut *(v as *mut [$int_type] as *mut [Self]) } - } - - /// Consumes the atomic and returns the contained value. - /// - /// This is safe because passing `self` by value guarantees that no other threads are - /// concurrently accessing the atomic data. - /// - /// # Examples - /// - #[cfg_attr($cfg_base, doc = "```")] - #[cfg_attr(not($cfg_base), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] - /// - #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] - /// assert_eq!(some_var.into_inner(), 5); - /// ``` - #[inline] - #[$stable_access] - #[$const_stable_into_inner] - pub const fn into_inner(self) -> $int_type { - // SAFETY: - // `Atomic` is essentially a transparent wrapper around `T`. - unsafe { transmute(self) } - } - - /// Loads a value from the atomic integer. - /// - /// `load` takes an [`Ordering`] argument which describes the memory ordering of this operation. - /// Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. - /// - /// # Panics - /// - /// Panics if `order` is [`Release`] or [`AcqRel`]. - /// - /// # Examples - /// - #[cfg_attr($cfg_base, doc = "```")] - #[cfg_attr(not($cfg_base), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] - /// - /// assert_eq!(some_var.load(Ordering::Relaxed), 5); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const fn load(&self, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_load::<_, /* VOLATILE */ false>(self.as_ptr(), order) } - } - - /// Perform a volatile load from the atomic integer. - /// - /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. - /// - #[doc = include_str!("./atomic_load_volatile.md")] - /// - /// # Safety - /// - /// Behavior is undefined if any of the following conditions are violated: - /// - /// * `self` must be [valid] for reads, or `self` must point to memory - /// outside of all Rust allocations and reading from that memory must: - /// - not trap, and - /// - not cause any memory inside a Rust allocation to be modified. - /// - /// * `self` must be aligned to - #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] - #[doc = if_8_bit!{ - $int_type, - yes = [ - " (note that this is always true, since `align_of::<", - stringify!($atomic_type), ">() == 1`)." - ], - no = [ - " (note that on some platforms this can be bigger than `align_of::<", - stringify!($int_type), ">()`)." - ], - }] - /// - /// * Reading from `self` must produce a properly initialized value of the underlying - /// integer type. - /// - /// [valid]: core::ptr#safety - /// - /// # Panics - /// - /// Panics if `order` is [`Release`] or [`AcqRel`]. - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> $int_type { - // SAFETY: follows from our own safety requirements. - unsafe { - atomic_load::<_, /* VOLATILE */ true>(self.cast::<$int_type>(), order) - } - } - - /// Stores a value into the atomic integer. - /// - /// `store` takes an [`Ordering`] argument which describes the memory ordering of this operation. - /// Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. - /// - /// # Panics - /// - /// Panics if `order` is [`Acquire`] or [`AcqRel`]. - /// - /// # Examples - /// - #[cfg_attr($cfg_base, doc = "```")] - #[cfg_attr(not($cfg_base), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] - /// - /// some_var.store(10, Ordering::Relaxed); - /// assert_eq!(some_var.load(Ordering::Relaxed), 10); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn store(&self, val: $int_type, order: Ordering) { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_store::<_, /* VOLATILE */ false>(self.as_ptr(), val, order); } - } - - /// Performs a volatile store into the atomic integer. - /// - /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. - /// - #[doc = include_str!("./atomic_store_volatile.md")] - /// - /// # Safety - /// - /// Behavior is undefined if any of the following conditions are violated: - /// - /// * `self` must be either [valid] for writes, or `self` must point to memory - /// outside of all Rust allocations and writing to that memory must: - /// - not trap, and - /// - not cause any memory inside a Rust allocation to be modified. - /// - /// * `self` must be aligned to - #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] - #[doc = if_8_bit!{ - $int_type, - yes = [ - " (note that this is always true, since `align_of::<", - stringify!($atomic_type), ">() == 1`)." - ], - no = [ - " (note that on some platforms this can be bigger than `align_of::<", - stringify!($int_type), ">()`)." - ], - }] - /// - /// [valid]: core::ptr#safety - /// - /// # Panics - /// - /// Panics if `order` is [`Acquire`] or [`AcqRel`]. - #[inline] - #[unstable(feature = "atomic_volatile", issue = "158947")] - #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const unsafe fn store_volatile(self: *const Self, val: $int_type, order: Ordering) { - // SAFETY: follows from our own safety requirements. - unsafe { - atomic_store::<_, /* VOLATILE */ true>(self.cast::<$int_type>().cast_mut(), val, order); - } - } - - /// Stores a value into the atomic integer, returning the previous value. - /// - /// `swap` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] - /// - /// assert_eq!(some_var.swap(10, Ordering::Relaxed), 5); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn swap(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_swap(self.as_ptr(), val, order) } - } - - /// Stores a value into the atomic integer if the current value is the same as - /// the `current` value. - /// - /// The return value is always the previous value. If it is equal to `current`, then the - /// value was updated. - /// - /// `compare_and_swap` also takes an [`Ordering`] argument which describes the memory - /// ordering of this operation. Notice that even when using [`AcqRel`], the operation - /// might fail and hence just perform an `Acquire` load, but not have `Release` semantics. - /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it - /// happens, and using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Migrating to `compare_exchange` and `compare_exchange_weak` - /// - /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for - /// memory orderings: - /// - /// Original | Success | Failure - /// -------- | ------- | ------- - /// Relaxed | Relaxed | Relaxed - /// Acquire | Acquire | Acquire - /// Release | Release | Relaxed - /// AcqRel | AcqRel | Acquire - /// SeqCst | SeqCst | SeqCst - /// - /// `compare_and_swap` and `compare_exchange` also differ in their return type. You can use - /// `compare_exchange(...).unwrap_or_else(|x| x)` to recover the behavior of `compare_and_swap`, - /// but in most cases it is more idiomatic to check whether the return value is `Ok` or `Err` - /// rather than to infer success vs failure based on the value that was read. - /// - /// During migration, consider whether it makes sense to use `compare_exchange_weak` instead. - /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, - /// which allows the compiler to generate better assembly code when the compare and swap - /// is used in a loop. - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] - /// - /// assert_eq!(some_var.compare_and_swap(5, 10, Ordering::Relaxed), 5); - /// assert_eq!(some_var.load(Ordering::Relaxed), 10); - /// - /// assert_eq!(some_var.compare_and_swap(6, 12, Ordering::Relaxed), 10); - /// assert_eq!(some_var.load(Ordering::Relaxed), 10); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[deprecated( - since = "1.50.0", - note = "Use `compare_exchange` or `compare_exchange_weak` instead") - ] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn compare_and_swap(&self, - current: $int_type, - new: $int_type, - order: Ordering) -> $int_type { - match self.compare_exchange(current, - new, - order, - strongest_failure_ordering(order)) { - Ok(x) => x, - Err(x) => x, - } - } - - /// Stores a value into the atomic integer if the current value is the same as - /// the `current` value. - /// - /// The return value is a result indicating whether the new value was written and - /// containing the previous value. On success this value is guaranteed to be equal to - /// `current`. - /// - /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. `success` describes the required ordering for the - /// read-modify-write operation that takes place if the comparison with `current` succeeds. - /// `failure` describes the required ordering for the load operation that takes place when - /// the comparison fails. Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] - /// - /// assert_eq!(some_var.compare_exchange(5, 10, - /// Ordering::Acquire, - /// Ordering::Relaxed), - /// Ok(5)); - /// assert_eq!(some_var.load(Ordering::Relaxed), 10); - /// - /// assert_eq!(some_var.compare_exchange(6, 12, - /// Ordering::SeqCst, - /// Ordering::Acquire), - /// Err(10)); - /// assert_eq!(some_var.load(Ordering::Relaxed), 10); - /// ``` - /// - /// # Considerations - /// - /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides - /// of CAS operations. In particular, a load of the value followed by a successful - /// `compare_exchange` with the previous load *does not ensure* that other threads have not - /// changed the value in the interim! This is usually important when the *equality* check in - /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. This is a particularly common case for pointers, as - /// a pointer holding the same address does not imply that the same object exists at that - /// address! In this case, `compare_exchange` can lead to the [ABA problem]. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap - #[inline] - #[$stable_cxchg] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn compare_exchange(&self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering) -> Result<$int_type, $int_type> { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_compare_exchange(self.as_ptr(), current, new, success, failure) } - } - - /// Stores a value into the atomic integer if the current value is the same as - /// the `current` value. - /// - #[doc = concat!("Unlike [`", stringify!($atomic_type), "::compare_exchange`],")] - /// this function is allowed to spuriously fail even - /// when the comparison succeeds, which can result in more efficient code on some - /// platforms. The return value is a result indicating whether the new value was - /// written and containing the previous value. - /// - /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory - /// ordering of this operation. `success` describes the required ordering for the - /// read-modify-write operation that takes place if the comparison with `current` succeeds. - /// `failure` describes the required ordering for the load operation that takes place when - /// the comparison fails. Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let val = ", stringify!($atomic_type), "::new(4);")] - /// - /// let mut old = val.load(Ordering::Relaxed); - /// loop { - /// let new = old * 2; - /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { - /// Ok(_) => break, - /// Err(x) => old = x, - /// } - /// } - /// ``` - /// - /// # Considerations - /// - /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides - /// of CAS operations. In particular, a load of the value followed by a successful - /// `compare_exchange` with the previous load *does not ensure* that other threads have not - /// changed the value in the interim. This is usually important when the *equality* check in - /// the `compare_exchange` is being used to check the *identity* of a value, but equality - /// does not necessarily imply identity. This is a particularly common case for pointers, as - /// a pointer holding the same address does not imply that the same object exists at that - /// address! In this case, `compare_exchange` can lead to the [ABA problem]. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap - #[inline] - #[$stable_cxchg] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn compare_exchange_weak(&self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering) -> Result<$int_type, $int_type> { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { - atomic_compare_exchange_weak(self.as_ptr(), current, new, success, failure) - } - } - - /// Adds to the current value, returning the previous value. - /// - /// This operation wraps around on overflow. - /// - /// `fetch_add` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0);")] - /// assert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0); - /// assert_eq!(foo.load(Ordering::SeqCst), 10); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_add(self.as_ptr(), val, order) } - } - - /// Subtracts from the current value, returning the previous value. - /// - /// This operation wraps around on overflow. - /// - /// `fetch_sub` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(20);")] - /// assert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20); - /// assert_eq!(foo.load(Ordering::SeqCst), 10); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_sub(self.as_ptr(), val, order) } - } - - /// Bitwise "and" with the current value. - /// - /// Performs a bitwise "and" operation on the current value and the argument `val`, and - /// sets the new value to the result. - /// - /// Returns the previous value. - /// - /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] - /// assert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101); - /// assert_eq!(foo.load(Ordering::SeqCst), 0b100001); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_and(self.as_ptr(), val, order) } - } - - /// Bitwise "nand" with the current value. - /// - /// Performs a bitwise "nand" operation on the current value and the argument `val`, and - /// sets the new value to the result. - /// - /// Returns the previous value. - /// - /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0x13);")] - /// assert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13); - /// assert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31)); - /// ``` - #[inline] - #[$stable_nand] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_nand(self.as_ptr(), val, order) } - } - - /// Bitwise "or" with the current value. - /// - /// Performs a bitwise "or" operation on the current value and the argument `val`, and - /// sets the new value to the result. - /// - /// Returns the previous value. - /// - /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] - /// assert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101); - /// assert_eq!(foo.load(Ordering::SeqCst), 0b111111); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_or(self.as_ptr(), val, order) } - } - - /// Bitwise "xor" with the current value. - /// - /// Performs a bitwise "xor" operation on the current value and the argument `val`, and - /// sets the new value to the result. - /// - /// Returns the previous value. - /// - /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] - /// assert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); - /// assert_eq!(foo.load(Ordering::SeqCst), 0b011110); - /// ``` - #[inline] - #[$stable] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_xor(self.as_ptr(), val, order) } - } - - /// An alias for - #[doc = concat!("[`", stringify!($atomic_type), "::try_update`]")] - /// . - #[inline] - #[stable(feature = "no_more_cas", since = "1.45.0")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - #[deprecated( - since = "1.99.0", - note = "renamed to `try_update` for consistency", - suggestion = "try_update" - )] - pub fn fetch_update(&self, - set_order: Ordering, - fetch_order: Ordering, - f: F) -> Result<$int_type, $int_type> - where F: FnMut($int_type) -> Option<$int_type> { - self.try_update(set_order, fetch_order, f) - } - - /// Fetches the value, and applies a function to it that returns an optional - /// new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else - /// `Err(previous_value)`. - /// - #[doc = concat!("See also: [`update`](`", stringify!($atomic_type), "::update`).")] - /// - /// Note: This may call the function multiple times if the value has been changed from other threads in - /// the meantime, as long as the function returns `Some(_)`, but the function will have been applied - /// only once to the stored value. - /// - /// `try_update` takes two [`Ordering`] arguments to describe the memory ordering of this operation. - /// The first describes the required ordering for when the operation finally succeeds while the second - /// describes the required ordering for loads. These correspond to the success and failure orderings of - #[doc = concat!("[`", stringify!($atomic_type), "::compare_exchange`]")] - /// respectively. - /// - /// Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load - /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Considerations - /// - /// This method is not magic; it is not provided by the hardware, and does not act like a - /// critical section or mutex. - /// - /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem] - /// if this atomic integer is an index or more generally if knowledge of only the *bitwise value* - /// of the atomic is not in and of itself sufficient to ensure any required preconditions. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```rust")] - #[cfg_attr(not($cfg_cas), doc = "```rust,compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let x = ", stringify!($atomic_type), "::new(7);")] - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(7)); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(7)); - /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(8)); - /// assert_eq!(x.load(Ordering::SeqCst), 9); - /// ``` - #[inline] - #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub fn try_update( - &self, - set_order: Ordering, - fetch_order: Ordering, - mut f: impl FnMut($int_type) -> Option<$int_type>, - ) -> Result<$int_type, $int_type> { - let mut prev = self.load(fetch_order); - while let Some(next) = f(prev) { - match self.compare_exchange_weak(prev, next, set_order, fetch_order) { - x @ Ok(_) => return x, - Err(next_prev) => prev = next_prev - } - } - Err(prev) - } - - /// Fetches the value, applies a function to it that it return a new value. - /// The new value is stored and the old value is returned. - /// - #[doc = concat!("See also: [`try_update`](`", stringify!($atomic_type), "::try_update`).")] - /// - /// Note: This may call the function multiple times if the value has been changed from other threads in - /// the meantime, but the function will have been applied only once to the stored value. - /// - /// `update` takes two [`Ordering`] arguments to describe the memory ordering of this operation. - /// The first describes the required ordering for when the operation finally succeeds while the second - /// describes the required ordering for loads. These correspond to the success and failure orderings of - #[doc = concat!("[`", stringify!($atomic_type), "::compare_exchange`]")] - /// respectively. - /// - /// Using [`Acquire`] as success ordering makes the store part - /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load - /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Considerations - /// - /// [CAS operation]: https://en.wikipedia.org/wiki/Compare-and-swap - /// This method is not magic; it is not provided by the hardware, and does not act like a - /// critical section or mutex. - /// - /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to - /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem] - /// if this atomic integer is an index or more generally if knowledge of only the *bitwise value* - /// of the atomic is not in and of itself sufficient to ensure any required preconditions. - /// - /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem - /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```rust")] - #[cfg_attr(not($cfg_cas), doc = "```rust,compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let x = ", stringify!($atomic_type), "::new(7);")] - /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| x + 1), 7); - /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| x + 1), 8); - /// assert_eq!(x.load(Ordering::SeqCst), 9); - /// ``` - #[inline] - #[stable(feature = "atomic_try_update", since = "1.95.0")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub fn update( - &self, - set_order: Ordering, - fetch_order: Ordering, - mut f: impl FnMut($int_type) -> $int_type, - ) -> $int_type { - let mut prev = self.load(fetch_order); - loop { - match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { - Ok(x) => break x, - Err(next_prev) => prev = next_prev, - } - } - } - - /// Maximum with the current value. - /// - /// Finds the maximum of the current value and the argument `val`, and - /// sets the new value to the result. - /// - /// Returns the previous value. - /// - /// `fetch_max` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] - /// assert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23); - /// assert_eq!(foo.load(Ordering::SeqCst), 42); - /// ``` - /// - /// If you want to obtain the maximum value in one step, you can use the following: - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] - /// let bar = 42; - /// let max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar); - /// assert!(max_foo == 42); - /// ``` - #[inline] - #[stable(feature = "atomic_min_max", since = "1.45.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { $max_fn(self.as_ptr(), val, order) } - } - - /// Minimum with the current value. - /// - /// Finds the minimum of the current value and the argument `val`, and - /// sets the new value to the result. - /// - /// Returns the previous value. - /// - /// `fetch_min` takes an [`Ordering`] argument which describes the memory ordering - /// of this operation. All ordering modes are possible. Note that using - /// [`Acquire`] makes the store part of this operation [`Relaxed`], and - /// using [`Release`] makes the load part [`Relaxed`]. - /// - /// **Note**: This method is only available on platforms that support atomic operations on - #[doc = concat!("[`", $s_int_type, "`].")] - /// - /// # Examples - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] - /// assert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23); - /// assert_eq!(foo.load(Ordering::Relaxed), 23); - /// assert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23); - /// assert_eq!(foo.load(Ordering::Relaxed), 22); - /// ``` - /// - /// If you want to obtain the minimum value in one step, you can use the following: - /// - #[cfg_attr($cfg_cas, doc = "```")] - #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] - #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] - /// - #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] - /// let bar = 12; - /// let min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar); - /// assert_eq!(min_foo, 12); - /// ``` - #[inline] - #[stable(feature = "atomic_min_max", since = "1.45.0")] - #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] - #[cfg(any($cfg_cas, doc))] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[rustc_should_not_be_called_on_const_items] - pub const fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { - // SAFETY: data races are prevented by atomic intrinsics. - unsafe { $min_fn(self.as_ptr(), val, order) } - } - - /// Returns a mutable pointer to the underlying integer. - /// - /// Doing non-atomic reads and writes on the resulting integer can be a data race. - /// This method is mostly useful for FFI, where the function signature may use - #[doc = concat!("`*mut ", stringify!($int_type), "` instead of `&", stringify!($atomic_type), "`.")] - /// - /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the - /// atomic types work with interior mutability. All modifications of an atomic change the value - /// through a shared reference, and can do so safely as long as they use atomic operations. Any - /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the - /// requirements of the [memory model]. - /// - /// # Examples - /// - /// ```ignore (extern-declaration) - /// # fn main() { - #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] - /// - /// extern "C" { - #[doc = concat!(" fn my_atomic_op(arg: *mut ", stringify!($int_type), ");")] - /// } - /// - #[doc = concat!("let atomic = ", stringify!($atomic_type), "::new(1);")] - /// - /// // SAFETY: Safe as long as `my_atomic_op` is atomic. - /// unsafe { - /// my_atomic_op(atomic.as_ptr()); - /// } - /// # } - /// ``` - /// - /// [memory model]: self#memory-model-for-atomic-accesses - #[inline] - #[stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] - #[rustc_never_returns_null_ptr] - pub const fn as_ptr(&self) -> *mut $int_type { - self.v.get().cast() - } - } } } #[cfg(target_has_atomic_load_store = "8")] -atomic_int! { - target_has_atomic_load_store = "8", - target_has_atomic = "8", - target_has_atomic_primitive_alignment = "8", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i8", - "", - atomic_min, atomic_max, - 1, - i8 AtomicI8 -} +atomic_int! { "i8", i8, AtomicI8 } #[cfg(target_has_atomic_load_store = "8")] -atomic_int! { - target_has_atomic_load_store = "8", - target_has_atomic = "8", - target_has_atomic_primitive_alignment = "8", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u8", - "", - atomic_umin, atomic_umax, - 1, - u8 AtomicU8 -} +atomic_int! { "u8", u8, AtomicU8 } #[cfg(target_has_atomic_load_store = "16")] -atomic_int! { - target_has_atomic_load_store = "16", - target_has_atomic = "16", - target_has_atomic_primitive_alignment = "16", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i16", - "", - atomic_min, atomic_max, - 2, - i16 AtomicI16 -} +atomic_int! { "i16", i16, AtomicI16 } #[cfg(target_has_atomic_load_store = "16")] -atomic_int! { - target_has_atomic_load_store = "16", - target_has_atomic = "16", - target_has_atomic_primitive_alignment = "16", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u16", - "", - atomic_umin, atomic_umax, - 2, - u16 AtomicU16 -} +atomic_int! { "u16", u16, AtomicU16 } #[cfg(target_has_atomic_load_store = "32")] -atomic_int! { - target_has_atomic_load_store = "32", - target_has_atomic = "32", - target_has_atomic_primitive_alignment = "32", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i32", - "", - atomic_min, atomic_max, - 4, - i32 AtomicI32 -} +atomic_int! { "i32", i32, AtomicI32 } #[cfg(target_has_atomic_load_store = "32")] -atomic_int! { - target_has_atomic_load_store = "32", - target_has_atomic = "32", - target_has_atomic_primitive_alignment = "32", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u32", - "", - atomic_umin, atomic_umax, - 4, - u32 AtomicU32 -} +atomic_int! { "u32", u32, AtomicU32 } #[cfg(target_has_atomic_load_store = "64")] -atomic_int! { - target_has_atomic_load_store = "64", - target_has_atomic = "64", - target_has_atomic_primitive_alignment = "64", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "i64", - "", - atomic_min, atomic_max, - 8, - i64 AtomicI64 -} +atomic_int! { "i64", i64, AtomicI64 } #[cfg(target_has_atomic_load_store = "64")] -atomic_int! { - target_has_atomic_load_store = "64", - target_has_atomic = "64", - target_has_atomic_primitive_alignment = "64", - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - stable(feature = "integer_atomics_stable", since = "1.34.0"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "u64", - "", - atomic_umin, atomic_umax, - 8, - u64 AtomicU64 -} +atomic_int! { "u64", u64, AtomicU64 } #[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { - target_has_atomic_load_store = "128", - target_has_atomic = "128", - target_has_atomic_primitive_alignment = "128", - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_unstable(feature = "integer_atomics", issue = "99069"), - "i128", - "#![feature(integer_atomics)]\n\n", - atomic_min, atomic_max, - 16, - i128 AtomicI128 -} +atomic_int! { "i128", i128, AtomicI128 } #[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { - target_has_atomic_load_store = "128", - target_has_atomic = "128", - target_has_atomic_primitive_alignment = "128", - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_unstable(feature = "integer_atomics", issue = "99069"), - "u128", - "#![feature(integer_atomics)]\n\n", - atomic_umin, atomic_umax, - 16, - u128 AtomicU128 -} +atomic_int! { "u128", u128, AtomicU128 } #[cfg(target_has_atomic_load_store = "ptr")] macro_rules! atomic_int_ptr_sized { ( $($target_pointer_width:literal $align:literal)* ) => { $( #[cfg(target_pointer_width = $target_pointer_width)] - atomic_int! { - target_has_atomic_load_store = "ptr", - target_has_atomic = "ptr", - target_has_atomic_primitive_alignment = "ptr", - stable(feature = "rust1", since = "1.0.0"), - stable(feature = "extended_compare_and_swap", since = "1.10.0"), - stable(feature = "atomic_debug", since = "1.3.0"), - stable(feature = "atomic_access", since = "1.15.0"), - stable(feature = "atomic_from", since = "1.23.0"), - stable(feature = "atomic_nand", since = "1.27.0"), - rustc_const_stable(feature = "const_ptr_sized_atomics", since = "1.24.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "isize", - "", - atomic_min, atomic_max, - $align, - isize AtomicIsize - } + atomic_int! { "isize", isize, AtomicIsize } #[cfg(target_pointer_width = $target_pointer_width)] - atomic_int! { - target_has_atomic_load_store = "ptr", - target_has_atomic = "ptr", - target_has_atomic_primitive_alignment = "ptr", - stable(feature = "rust1", since = "1.0.0"), - stable(feature = "extended_compare_and_swap", since = "1.10.0"), - stable(feature = "atomic_debug", since = "1.3.0"), - stable(feature = "atomic_access", since = "1.15.0"), - stable(feature = "atomic_from", since = "1.23.0"), - stable(feature = "atomic_nand", since = "1.27.0"), - rustc_const_stable(feature = "const_ptr_sized_atomics", since = "1.24.0"), - rustc_const_stable(feature = "const_atomic_into_inner", since = "1.79.0"), - "usize", - "", - atomic_umin, atomic_umax, - $align, - usize AtomicUsize - } + atomic_int! { "usize", usize, AtomicUsize } /// An [`AtomicIsize`] initialized to `0`. #[cfg(target_pointer_width = $target_pointer_width)] @@ -4259,7 +2936,6 @@ macro_rules! atomic_int_ptr_sized { note = "the `new` function is now preferred", suggestion = "AtomicIsize::new(0)", )] - #[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] pub const ATOMIC_ISIZE_INIT: AtomicIsize = AtomicIsize::new(0); /// An [`AtomicUsize`] initialized to `0`. @@ -4270,7 +2946,6 @@ macro_rules! atomic_int_ptr_sized { note = "the `new` function is now preferred", suggestion = "AtomicUsize::new(0)", )] - #[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] pub const ATOMIC_USIZE_INIT: AtomicUsize = AtomicUsize::new(0); )* }; } From 2df8fc2c56d990e2391025bc2037fd1b536c2526 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 00:37:19 -0400 Subject: [PATCH 02/16] formatting fixes --- library/core/src/sync/atomic.rs | 83 +++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 21bc5d9796d9e..e33f228b14d7f 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -354,7 +354,7 @@ pub impl(self) unsafe trait AtomicCas: AtomicPrimitive {} /// Types implementing this trait must be primitives whose alignment is the same as their atomic /// counter parts. Implementing this trait automatically implements the following methods for /// [Atomic]\: -/// +/// /// - from_mut /// - from_mut_slice #[unstable( @@ -373,7 +373,7 @@ pub impl(self) unsafe trait AtomicAlignedPrimitive: AtomicPrimitive {} /// /// Types implementing this trait must be primitive integer types. Implementing this trait /// automatically implements the following atomic methods for [Atomic]\: -/// +/// /// - fetch_add /// - fetch_sub /// - fetch_max @@ -540,7 +540,13 @@ impl_atomic_traits!(i64 as Align8, size("64"), load_store, cas, aligned, si impl_atomic_traits!(u64 as Align8, size("64"), load_store, cas, aligned, unsigned, bitwise); impl_atomic_traits!(i128 as Align16, size("128"), load_store, cas, aligned, signed, bitwise); impl_atomic_traits!( - u128 as Align16, size("128"), load_store, cas, aligned, unsigned, bitwise + u128 as Align16, + size("128"), + load_store, + cas, + aligned, + unsigned, + bitwise ); #[cfg(target_pointer_width = "16")] @@ -552,15 +558,33 @@ impl_atomic_traits!(isize as Align8, size("ptr"), load_store, cas, aligne #[cfg(target_pointer_width = "16")] impl_atomic_traits!( - usize as Align2, size("ptr"), load_store, cas, aligned, unsigned, bitwise + usize as Align2, + size("ptr"), + load_store, + cas, + aligned, + unsigned, + bitwise ); #[cfg(target_pointer_width = "32")] impl_atomic_traits!( - usize as Align4, size("ptr"), load_store, cas, aligned, unsigned, bitwise + usize as Align4, + size("ptr"), + load_store, + cas, + aligned, + unsigned, + bitwise ); #[cfg(target_pointer_width = "64")] impl_atomic_traits!( - usize as Align8, size("ptr"), load_store, cas, aligned, unsigned, bitwise + usize as Align8, + size("ptr"), + load_store, + cas, + aligned, + unsigned, + bitwise ); #[cfg(target_pointer_width = "16")] @@ -619,7 +643,7 @@ const EMULATE_ATOMIC_BOOL: bool = cfg!(any( /// loads and stores of `u8`. #[cfg(target_has_atomic_load_store = "8")] #[stable(feature = "rust1", since = "1.0.0")] -pub type AtomicBool = Atomic::; +pub type AtomicBool = Atomic; /// A raw pointer type which can be safely shared between threads. /// @@ -722,16 +746,15 @@ impl Atomic { /// /// let atomic = Atomic:::new(true); /// ``` - #[inline] - #[stable(feature = "integer_atomics_stable", since="1.34.0")] - #[rustc_const_stable(feature= "const_atomic_new", since="1.34.0")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + #[rustc_const_stable(feature = "const_atomic_new", since = "1.34.0")] #[must_use] pub const fn new(v: T) -> Self { Self { // SAFETY: // By the contract of AtomicPrimitive, it is guaranteed that transmuting between T and // T::Storage is valid. - v: UnsafeCell::new(unsafe { transmute_unchecked(v) }) + v: UnsafeCell::new(unsafe { transmute_unchecked(v) }), } } @@ -940,9 +963,10 @@ impl Atomic { // SAFETY: any data races are prevented by atomic intrinsics and the raw // pointer passed in is valid because we got it from a reference. unsafe { - transmute_unchecked( - atomic_load::<_, /* VOLATILE */ false>(self.v.get().cast::(), order) - ) + transmute_unchecked(atomic_load::<_, /* VOLATILE */ false>( + self.v.get().cast::(), + order, + )) } } @@ -1003,9 +1027,10 @@ impl Atomic { pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> T { // SAFETY: follows from our own safety requirements. unsafe { - transmute_unchecked( - atomic_load::<_, /* VOLATILE */ true>(self.cast::(), order) - ) + transmute_unchecked(atomic_load::<_, /* VOLATILE */ true>( + self.cast::(), + order, + )) } } @@ -1038,7 +1063,9 @@ impl Atomic { // pointer passed in is valid because we got it from a reference. unsafe { atomic_store::<_, /* VOLATILE */ false>( - self.v.get().cast::(), transmute_unchecked::<_, T::OpType>(val), order + self.v.get().cast::(), + transmute_unchecked::<_, T::OpType>(val), + order, ); } } @@ -1098,7 +1125,9 @@ impl Atomic { // SAFETY: follows from our own safety requirements. unsafe { atomic_store::<_, /* VOLATILE */ true>( - self.cast::().cast_mut(), transmute_unchecked::<_, T::OpType>(val), order + self.cast::().cast_mut(), + transmute_unchecked::<_, T::OpType>(val), + order, ); } } @@ -1127,7 +1156,7 @@ impl Atomic { #[rustc_should_not_be_called_on_const_items] pub const fn swap(&self, v: T, order: Ordering) -> T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_swap(self.as_ptr(), v, order) } + unsafe { atomic_swap(self.as_ptr(), v, order) } } /// Stores a value into the pointer if the current value is the same as the `current` value. @@ -1319,12 +1348,7 @@ impl Atomic { note = "renamed to `try_update` for consistency", suggestion = "try_update" )] - pub fn fetch_update( - &self, - set_order: Ordering, - fetch_order: Ordering, - f: F, - ) -> Result + pub fn fetch_update(&self, set_order: Ordering, fetch_order: Ordering, f: F) -> Result where F: FnMut(T) -> Option, { @@ -1643,8 +1667,7 @@ impl Atomic { // SAFETY: data races are prevented by atomic intrinsics. if T::IS_SIGNED { unsafe { atomic_max(self.as_ptr(), val, order) } - } - else { + } else { unsafe { atomic_umax(self.as_ptr(), val, order) } } } @@ -1692,8 +1715,7 @@ impl Atomic { // SAFETY: data races are prevented by atomic intrinsics. if ::IS_SIGNED { unsafe { atomic_min(self.as_ptr(), val, order) } - } - else { + } else { unsafe { atomic_umin(self.as_ptr(), val, order) } } } @@ -2876,7 +2898,6 @@ macro_rules! atomic_int { "lesser alignment." ], }] - /// /// For more about the differences between atomic types and /// non-atomic types as well as information about the portability of /// this type, please see the [module-level documentation]. From 232f8504a359ab2d1dcc8f410ce5d8ef1bf5c757 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 01:29:59 -0400 Subject: [PATCH 03/16] undo impls for 128 bit atomics since unstable impls don't exist --- library/core/src/sync/atomic.rs | 1299 ++++++++++++++++++++++++++++++- 1 file changed, 1287 insertions(+), 12 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index e33f228b14d7f..fba1e609f9865 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -538,16 +538,28 @@ impl_atomic_traits!(i32 as Align4, size("32"), load_store, cas, aligned, si impl_atomic_traits!(u32 as Align4, size("32"), load_store, cas, aligned, unsigned, bitwise); impl_atomic_traits!(i64 as Align8, size("64"), load_store, cas, aligned, signed, bitwise); impl_atomic_traits!(u64 as Align8, size("64"), load_store, cas, aligned, unsigned, bitwise); -impl_atomic_traits!(i128 as Align16, size("128"), load_store, cas, aligned, signed, bitwise); -impl_atomic_traits!( - u128 as Align16, - size("128"), - load_store, - cas, - aligned, - unsigned, - bitwise -); + +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +#[cfg(any(target_has_atomic_load_store = "128", doc))] +unsafe impl AtomicPrimitive for i128 { + type Storage = private::Align16; + type OpType = i128; +} + +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +#[cfg(any(target_has_atomic_load_store = "128", doc))] +unsafe impl AtomicPrimitive for u128 { + type Storage = private::Align16; + type OpType = u128; +} #[cfg(target_pointer_width = "16")] impl_atomic_traits!(isize as Align2, size("ptr"), load_store, cas, aligned, signed, bitwise); @@ -2936,10 +2948,1273 @@ atomic_int! { "u32", u32, AtomicU32 } atomic_int! { "i64", i64, AtomicI64 } #[cfg(target_has_atomic_load_store = "64")] atomic_int! { "u64", u64, AtomicU64 } + +#[cfg(target_has_atomic_load_store)] +macro_rules! atomic_int_legacy { + ($cfg_base:meta, + $cfg_cas:meta, + $cfg_align:meta, + $stable:meta, + $stable_cxchg:meta, + $stable_debug:meta, + $stable_access:meta, + $stable_from:meta, + $stable_nand:meta, + $const_stable_new:meta, + $const_stable_into_inner:meta, + $s_int_type:literal, + $extra_feature:expr, + $min_fn:ident, $max_fn:ident, + $align:expr, + $int_type:ident $atomic_type:ident) => { + /// An integer type which can be safely shared between threads. + /// + /// This type has the same + #[doc = if_8_bit!( + $int_type, + yes = ["size, alignment, and bit validity"], + no = ["size and bit validity"], + )] + /// as the underlying integer type, [` + #[doc = $s_int_type] + /// `]. + #[doc = if_8_bit! { + $int_type, + no = [ + "However, the alignment of this type is always equal to its ", + "size, even on targets where [`", $s_int_type, "`] has a ", + "lesser alignment." + ], + }] + /// + /// For more about the differences between atomic types and + /// non-atomic types as well as information about the portability of + /// this type, please see the [module-level documentation]. + /// + /// **Note:** This type is only available on platforms that support + /// atomic loads and stores of [` + #[doc = $s_int_type] + /// `]. + /// + /// [module-level documentation]: crate::sync::atomic + #[$stable] + pub type $atomic_type = Atomic<$int_type>; + + #[$stable] + impl Default for $atomic_type { + #[inline] + fn default() -> Self { + Self::new(Default::default()) + } + } + + #[$stable_from] + #[rustc_const_unstable(feature = "const_convert", issue = "143773")] + const impl From<$int_type> for $atomic_type { + #[doc = concat!("Converts an `", stringify!($int_type), "` into an `", stringify!($atomic_type), "`.")] + #[inline] + fn from(v: $int_type) -> Self { Self::new(v) } + } + + #[$stable_debug] + impl fmt::Debug for $atomic_type { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.load(Ordering::Relaxed), f) + } + } + + impl $atomic_type { + /// Creates a new atomic integer. + /// + /// # Examples + /// + #[cfg_attr($cfg_base, doc = "```")] + #[cfg_attr(not($cfg_base), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] + /// + #[doc = concat!("let atomic_forty_two = ", stringify!($atomic_type), "::new(42);")] + /// ``` + #[inline] + #[$stable] + #[$const_stable_new] + #[must_use] + pub const fn new(v: $int_type) -> Self { + // SAFETY: + // `Atomic` is essentially a transparent wrapper around `T`. + unsafe { transmute(v) } + } + + /// Creates a new reference to an atomic integer from a pointer. + /// + /// # Examples + /// + #[cfg_attr($cfg_base, doc = "```rust")] + #[cfg_attr(not($cfg_base), doc = "```rust,compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{self, ", stringify!($atomic_type), "};")] + /// + /// // Get a pointer to an allocated value + #[doc = concat!("let ptr: *mut ", stringify!($int_type), " = Box::into_raw(Box::new(0));")] + /// + #[doc = concat!("assert!(ptr.cast::<", stringify!($atomic_type), ">().is_aligned());")] + /// + /// { + /// // Create an atomic view of the allocated value + // SAFETY: this is a doc comment, tidy, it can't hurt you (also guaranteed by the construction of `ptr` and the assert above) + #[doc = concat!(" let atomic = unsafe {", stringify!($atomic_type), "::from_ptr(ptr) };")] + /// + /// // Use `atomic` for atomic operations, possibly share it with other threads + /// atomic.store(1, atomic::Ordering::Relaxed); + /// } + /// + /// // It's ok to non-atomically access the value behind `ptr`, + /// // since the reference to the atomic ended its lifetime in the block above + /// assert_eq!(unsafe { *ptr }, 1); + /// + /// // Deallocate the value + /// unsafe { drop(Box::from_raw(ptr)) } + /// ``` + /// + /// # Safety + /// + /// * `ptr` must be aligned to + #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] + #[doc = if_8_bit!{ + $int_type, + yes = [ + " (note that this is always true, since `align_of::<", + stringify!($atomic_type), ">() == 1`)." + ], + no = [ + " (note that on some platforms this can be bigger than `align_of::<", + stringify!($int_type), ">()`)." + ], + }] + /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`. + /// * You must adhere to the [Memory model for atomic accesses]. In particular, it is not + /// allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different + /// sizes, without synchronization. + /// + /// [valid]: crate::ptr#safety + /// [Memory model for atomic accesses]: self#memory-model-for-atomic-accesses + #[inline] + #[stable(feature = "atomic_from_ptr", since = "1.75.0")] + #[rustc_const_stable(feature = "const_atomic_from_ptr", since = "1.84.0")] + pub const unsafe fn from_ptr<'a>(ptr: *mut $int_type) -> &'a $atomic_type { + // SAFETY: guaranteed by the caller + unsafe { &*ptr.cast() } + } + + /// Creates a new pointer to an atomic integer from a pointer. + /// + /// This is useful if you want to do volatile atomic accesses, and thus avoid creating + /// a reference to the destination. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + pub const fn from_ptr_raw(ptr: *mut $int_type) -> *const $atomic_type { + ptr.cast_const().cast() + } + + /// Returns a mutable reference to the underlying integer. + /// + /// This is safe because the mutable reference guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + #[cfg_attr($cfg_base, doc = "```")] + #[cfg_attr(not($cfg_base), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let mut some_var = ", stringify!($atomic_type), "::new(10);")] + /// assert_eq!(*some_var.get_mut(), 10); + /// *some_var.get_mut() = 5; + /// assert_eq!(some_var.load(Ordering::SeqCst), 5); + /// ``` + #[inline] + #[$stable_access] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut(&mut self) -> &mut $int_type { + // SAFETY: + // `Atomic` is essentially a transparent wrapper around `T`. + unsafe { &mut *self.as_ptr() } + } + + #[doc = concat!("Get atomic access to a `&mut ", stringify!($int_type), "`.")] + /// + #[doc = if_8_bit! { + $int_type, + no = [ + "**Note:** This function is only available on targets where `", + stringify!($atomic_type), "` has the same alignment as `", stringify!($int_type), "`." + ], + }] + /// + /// # Examples + /// + #[cfg_attr($cfg_align, doc = "```rust")] + #[cfg_attr(not($cfg_align), doc = "```rust,compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + /// let mut some_int = 123; + #[doc = concat!("let a = ", stringify!($atomic_type), "::from_mut(&mut some_int);")] + /// a.store(100, Ordering::Relaxed); + /// assert_eq!(some_int, 100); + /// ``` + /// + #[inline] + #[cfg(any($cfg_align, doc))] + #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut(v: &mut $int_type) -> &mut Self { + let [] = [(); align_of::() - align_of::<$int_type>()]; + // SAFETY: + // - the mutable reference guarantees unique ownership. + // - the alignment of `$int_type` and `Self` is the + // same, as promised by $cfg_align and verified above. + unsafe { &mut *(v as *mut $int_type as *mut Self) } + } + + #[doc = concat!("Get non-atomic access to a `&mut [", stringify!($atomic_type), "]` slice")] + /// + /// This is safe because the mutable reference guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + #[cfg_attr($cfg_base, doc = "```ignore-wasm")] + #[cfg_attr(not($cfg_base), doc = "```ignore-wasm,compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let mut some_ints = [const { ", stringify!($atomic_type), "::new(0) }; 10];")] + /// + #[doc = concat!("let view: &mut [", stringify!($int_type), "] = ", stringify!($atomic_type), "::get_mut_slice(&mut some_ints);")] + /// assert_eq!(view, [0; 10]); + /// view + /// .iter_mut() + /// .enumerate() + /// .for_each(|(idx, int)| *int = idx as _); + /// + /// std::thread::scope(|s| { + /// some_ints + /// .iter() + /// .enumerate() + /// .for_each(|(idx, int)| { + /// s.spawn(move || assert_eq!(int.load(Ordering::Relaxed), idx as _)); + /// }) + /// }); + /// ``` + #[inline] + #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn get_mut_slice(this: &mut [Self]) -> &mut [$int_type] { + // SAFETY: the mutable reference guarantees unique ownership. + unsafe { &mut *(this as *mut [Self] as *mut [$int_type]) } + } + + #[doc = concat!("Get atomic access to a `&mut [", stringify!($int_type), "]` slice.")] + /// + #[doc = if_8_bit! { + $int_type, + no = [ + "**Note:** This function is only available on targets where `", + stringify!($atomic_type), "` has the same alignment as `", stringify!($int_type), "`." + ], + }] + /// + /// # Examples + /// + #[cfg_attr($cfg_align, doc = "```ignore-wasm")] + #[cfg_attr(not($cfg_align), doc = "```ignore-wasm,compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + /// let mut some_ints = [0; 10]; + #[doc = concat!("let a = &*", stringify!($atomic_type), "::from_mut_slice(&mut some_ints);")] + /// std::thread::scope(|s| { + /// for i in 0..a.len() { + /// s.spawn(move || a[i].store(i as _, Ordering::Relaxed)); + /// } + /// }); + /// for (i, n) in some_ints.into_iter().enumerate() { + /// assert_eq!(i, n as usize); + /// } + /// ``` + #[inline] + #[cfg(any($cfg_align, doc))] + #[stable(feature = "atomic_from_mut", since = "1.98.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + pub const fn from_mut_slice(v: &mut [$int_type]) -> &mut [Self] { + let [] = [(); align_of::() - align_of::<$int_type>()]; + // SAFETY: + // - the mutable reference guarantees unique ownership. + // - the alignment of `$int_type` and `Self` is the + // same, as promised by $cfg_align and verified above. + unsafe { &mut *(v as *mut [$int_type] as *mut [Self]) } + } + + /// Consumes the atomic and returns the contained value. + /// + /// This is safe because passing `self` by value guarantees that no other threads are + /// concurrently accessing the atomic data. + /// + /// # Examples + /// + #[cfg_attr($cfg_base, doc = "```")] + #[cfg_attr(not($cfg_base), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// assert_eq!(some_var.into_inner(), 5); + /// ``` + #[inline] + #[$stable_access] + #[$const_stable_into_inner] + pub const fn into_inner(self) -> $int_type { + // SAFETY: + // `Atomic` is essentially a transparent wrapper around `T`. + unsafe { transmute(self) } + } + + /// Loads a value from the atomic integer. + /// + /// `load` takes an [`Ordering`] argument which describes the memory ordering of this operation. + /// Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// + /// # Panics + /// + /// Panics if `order` is [`Release`] or [`AcqRel`]. + /// + /// # Examples + /// + #[cfg_attr($cfg_base, doc = "```")] + #[cfg_attr(not($cfg_base), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.load(Ordering::Relaxed), 5); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const fn load(&self, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_load::<_, /* VOLATILE */ false>(self.as_ptr(), order) } + } + + /// Perform a volatile load from the atomic integer. + /// + /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_load_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be [valid] for reads, or `self` must point to memory + /// outside of all Rust allocations and reading from that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// * `self` must be aligned to + #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] + #[doc = if_8_bit!{ + $int_type, + yes = [ + " (note that this is always true, since `align_of::<", + stringify!($atomic_type), ">() == 1`)." + ], + no = [ + " (note that on some platforms this can be bigger than `align_of::<", + stringify!($int_type), ">()`)." + ], + }] + /// + /// * Reading from `self` must produce a properly initialized value of the underlying + /// integer type. + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Release`] or [`AcqRel`]. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> $int_type { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_load::<_, /* VOLATILE */ true>(self.cast::<$int_type>(), order) + } + } + + /// Stores a value into the atomic integer. + /// + /// `store` takes an [`Ordering`] argument which describes the memory ordering of this operation. + /// Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. + /// + /// # Panics + /// + /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// + /// # Examples + /// + #[cfg_attr($cfg_base, doc = "```")] + #[cfg_attr(not($cfg_base), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// some_var.store(10, Ordering::Relaxed); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn store(&self, val: $int_type, order: Ordering) { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_store::<_, /* VOLATILE */ false>(self.as_ptr(), val, order); } + } + + /// Performs a volatile store into the atomic integer. + /// + /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_store_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be either [valid] for writes, or `self` must point to memory + /// outside of all Rust allocations and writing to that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// * `self` must be aligned to + #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] + #[doc = if_8_bit!{ + $int_type, + yes = [ + " (note that this is always true, since `align_of::<", + stringify!($atomic_type), ">() == 1`)." + ], + no = [ + " (note that on some platforms this can be bigger than `align_of::<", + stringify!($int_type), ">()`)." + ], + }] + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const unsafe fn store_volatile(self: *const Self, val: $int_type, order: Ordering) { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_store::<_, /* VOLATILE */ true>(self.cast::<$int_type>().cast_mut(), val, order); + } + } + + /// Stores a value into the atomic integer, returning the previous value. + /// + /// `swap` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.swap(10, Ordering::Relaxed), 5); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn swap(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_swap(self.as_ptr(), val, order) } + } + + /// Stores a value into the atomic integer if the current value is the same as + /// the `current` value. + /// + /// The return value is always the previous value. If it is equal to `current`, then the + /// value was updated. + /// + /// `compare_and_swap` also takes an [`Ordering`] argument which describes the memory + /// ordering of this operation. Notice that even when using [`AcqRel`], the operation + /// might fail and hence just perform an `Acquire` load, but not have `Release` semantics. + /// Using [`Acquire`] makes the store part of this operation [`Relaxed`] if it + /// happens, and using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Migrating to `compare_exchange` and `compare_exchange_weak` + /// + /// `compare_and_swap` is equivalent to `compare_exchange` with the following mapping for + /// memory orderings: + /// + /// Original | Success | Failure + /// -------- | ------- | ------- + /// Relaxed | Relaxed | Relaxed + /// Acquire | Acquire | Acquire + /// Release | Release | Relaxed + /// AcqRel | AcqRel | Acquire + /// SeqCst | SeqCst | SeqCst + /// + /// `compare_and_swap` and `compare_exchange` also differ in their return type. You can use + /// `compare_exchange(...).unwrap_or_else(|x| x)` to recover the behavior of `compare_and_swap`, + /// but in most cases it is more idiomatic to check whether the return value is `Ok` or `Err` + /// rather than to infer success vs failure based on the value that was read. + /// + /// During migration, consider whether it makes sense to use `compare_exchange_weak` instead. + /// `compare_exchange_weak` is allowed to fail spuriously even when the comparison succeeds, + /// which allows the compiler to generate better assembly code when the compare and swap + /// is used in a loop. + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.compare_and_swap(5, 10, Ordering::Relaxed), 5); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// + /// assert_eq!(some_var.compare_and_swap(6, 12, Ordering::Relaxed), 10); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[deprecated( + since = "1.50.0", + note = "Use `compare_exchange` or `compare_exchange_weak` instead") + ] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_and_swap(&self, + current: $int_type, + new: $int_type, + order: Ordering) -> $int_type { + match self.compare_exchange(current, + new, + order, + strongest_failure_ordering(order)) { + Ok(x) => x, + Err(x) => x, + } + } + + /// Stores a value into the atomic integer if the current value is the same as + /// the `current` value. + /// + /// The return value is a result indicating whether the new value was written and + /// containing the previous value. On success this value is guaranteed to be equal to + /// `current`. + /// + /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let some_var = ", stringify!($atomic_type), "::new(5);")] + /// + /// assert_eq!(some_var.compare_exchange(5, 10, + /// Ordering::Acquire, + /// Ordering::Relaxed), + /// Ok(5)); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// + /// assert_eq!(some_var.compare_exchange(6, 12, + /// Ordering::SeqCst, + /// Ordering::Acquire), + /// Err(10)); + /// assert_eq!(some_var.load(Ordering::Relaxed), 10); + /// ``` + /// + /// # Considerations + /// + /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides + /// of CAS operations. In particular, a load of the value followed by a successful + /// `compare_exchange` with the previous load *does not ensure* that other threads have not + /// changed the value in the interim! This is usually important when the *equality* check in + /// the `compare_exchange` is being used to check the *identity* of a value, but equality + /// does not necessarily imply identity. This is a particularly common case for pointers, as + /// a pointer holding the same address does not imply that the same object exists at that + /// address! In this case, `compare_exchange` can lead to the [ABA problem]. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + #[inline] + #[$stable_cxchg] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_exchange(&self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering) -> Result<$int_type, $int_type> { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_compare_exchange(self.as_ptr(), current, new, success, failure) } + } + + /// Stores a value into the atomic integer if the current value is the same as + /// the `current` value. + /// + #[doc = concat!("Unlike [`", stringify!($atomic_type), "::compare_exchange`],")] + /// this function is allowed to spuriously fail even + /// when the comparison succeeds, which can result in more efficient code on some + /// platforms. The return value is a result indicating whether the new value was + /// written and containing the previous value. + /// + /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the successful load + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let val = ", stringify!($atomic_type), "::new(4);")] + /// + /// let mut old = val.load(Ordering::Relaxed); + /// loop { + /// let new = old * 2; + /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { + /// Ok(_) => break, + /// Err(x) => old = x, + /// } + /// } + /// ``` + /// + /// # Considerations + /// + /// `compare_exchange` is a [compare-and-swap operation] and thus exhibits the usual downsides + /// of CAS operations. In particular, a load of the value followed by a successful + /// `compare_exchange` with the previous load *does not ensure* that other threads have not + /// changed the value in the interim. This is usually important when the *equality* check in + /// the `compare_exchange` is being used to check the *identity* of a value, but equality + /// does not necessarily imply identity. This is a particularly common case for pointers, as + /// a pointer holding the same address does not imply that the same object exists at that + /// address! In this case, `compare_exchange` can lead to the [ABA problem]. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + #[inline] + #[$stable_cxchg] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn compare_exchange_weak(&self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering) -> Result<$int_type, $int_type> { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { + atomic_compare_exchange_weak(self.as_ptr(), current, new, success, failure) + } + } + + /// Adds to the current value, returning the previous value. + /// + /// This operation wraps around on overflow. + /// + /// `fetch_add` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0);")] + /// assert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_add(self.as_ptr(), val, order) } + } + + /// Subtracts from the current value, returning the previous value. + /// + /// This operation wraps around on overflow. + /// + /// `fetch_sub` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(20);")] + /// assert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20); + /// assert_eq!(foo.load(Ordering::SeqCst), 10); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_sub(self.as_ptr(), val, order) } + } + + /// Bitwise "and" with the current value. + /// + /// Performs a bitwise "and" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] + /// assert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b100001); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_and(self.as_ptr(), val, order) } + } + + /// Bitwise "nand" with the current value. + /// + /// Performs a bitwise "nand" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0x13);")] + /// assert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13); + /// assert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31)); + /// ``` + #[inline] + #[$stable_nand] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_nand(self.as_ptr(), val, order) } + } + + /// Bitwise "or" with the current value. + /// + /// Performs a bitwise "or" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] + /// assert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b111111); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_or(self.as_ptr(), val, order) } + } + + /// Bitwise "xor" with the current value. + /// + /// Performs a bitwise "xor" operation on the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(0b101101);")] + /// assert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); + /// assert_eq!(foo.load(Ordering::SeqCst), 0b011110); + /// ``` + #[inline] + #[$stable] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { atomic_xor(self.as_ptr(), val, order) } + } + + /// An alias for + #[doc = concat!("[`", stringify!($atomic_type), "::try_update`]")] + /// . + #[inline] + #[stable(feature = "no_more_cas", since = "1.45.0")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + #[deprecated( + since = "1.99.0", + note = "renamed to `try_update` for consistency", + suggestion = "try_update" + )] + pub fn fetch_update(&self, + set_order: Ordering, + fetch_order: Ordering, + f: F) -> Result<$int_type, $int_type> + where F: FnMut($int_type) -> Option<$int_type> { + self.try_update(set_order, fetch_order, f) + } + + /// Fetches the value, and applies a function to it that returns an optional + /// new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else + /// `Err(previous_value)`. + /// + #[doc = concat!("See also: [`update`](`", stringify!($atomic_type), "::update`).")] + /// + /// Note: This may call the function multiple times if the value has been changed from other threads in + /// the meantime, as long as the function returns `Some(_)`, but the function will have been applied + /// only once to the stored value. + /// + /// `try_update` takes two [`Ordering`] arguments to describe the memory ordering of this operation. + /// The first describes the required ordering for when the operation finally succeeds while the second + /// describes the required ordering for loads. These correspond to the success and failure orderings of + #[doc = concat!("[`", stringify!($atomic_type), "::compare_exchange`]")] + /// respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Considerations + /// + /// This method is not magic; it is not provided by the hardware, and does not act like a + /// critical section or mutex. + /// + /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem] + /// if this atomic integer is an index or more generally if knowledge of only the *bitwise value* + /// of the atomic is not in and of itself sufficient to ensure any required preconditions. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```rust")] + #[cfg_attr(not($cfg_cas), doc = "```rust,compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let x = ", stringify!($atomic_type), "::new(7);")] + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(7)); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(7)); + /// assert_eq!(x.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(8)); + /// assert_eq!(x.load(Ordering::SeqCst), 9); + /// ``` + #[inline] + #[stable(feature = "atomic_try_update", since = "1.95.0")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub fn try_update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: impl FnMut($int_type) -> Option<$int_type>, + ) -> Result<$int_type, $int_type> { + let mut prev = self.load(fetch_order); + while let Some(next) = f(prev) { + match self.compare_exchange_weak(prev, next, set_order, fetch_order) { + x @ Ok(_) => return x, + Err(next_prev) => prev = next_prev + } + } + Err(prev) + } + + /// Fetches the value, applies a function to it that it return a new value. + /// The new value is stored and the old value is returned. + /// + #[doc = concat!("See also: [`try_update`](`", stringify!($atomic_type), "::try_update`).")] + /// + /// Note: This may call the function multiple times if the value has been changed from other threads in + /// the meantime, but the function will have been applied only once to the stored value. + /// + /// `update` takes two [`Ordering`] arguments to describe the memory ordering of this operation. + /// The first describes the required ordering for when the operation finally succeeds while the second + /// describes the required ordering for loads. These correspond to the success and failure orderings of + #[doc = concat!("[`", stringify!($atomic_type), "::compare_exchange`]")] + /// respectively. + /// + /// Using [`Acquire`] as success ordering makes the store part + /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Considerations + /// + /// [CAS operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// This method is not magic; it is not provided by the hardware, and does not act like a + /// critical section or mutex. + /// + /// It is implemented on top of an atomic [compare-and-swap operation], and thus is subject to + /// the usual drawbacks of CAS operations. In particular, be careful of the [ABA problem] + /// if this atomic integer is an index or more generally if knowledge of only the *bitwise value* + /// of the atomic is not in and of itself sufficient to ensure any required preconditions. + /// + /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem + /// [compare-and-swap operation]: https://en.wikipedia.org/wiki/Compare-and-swap + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```rust")] + #[cfg_attr(not($cfg_cas), doc = "```rust,compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let x = ", stringify!($atomic_type), "::new(7);")] + /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| x + 1), 7); + /// assert_eq!(x.update(Ordering::SeqCst, Ordering::SeqCst, |x| x + 1), 8); + /// assert_eq!(x.load(Ordering::SeqCst), 9); + /// ``` + #[inline] + #[stable(feature = "atomic_try_update", since = "1.95.0")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub fn update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: impl FnMut($int_type) -> $int_type, + ) -> $int_type { + let mut prev = self.load(fetch_order); + loop { + match self.compare_exchange_weak(prev, f(prev), set_order, fetch_order) { + Ok(x) => break x, + Err(next_prev) => prev = next_prev, + } + } + } + + /// Maximum with the current value. + /// + /// Finds the maximum of the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_max` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// assert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23); + /// assert_eq!(foo.load(Ordering::SeqCst), 42); + /// ``` + /// + /// If you want to obtain the maximum value in one step, you can use the following: + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// let bar = 42; + /// let max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar); + /// assert!(max_foo == 42); + /// ``` + #[inline] + #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { $max_fn(self.as_ptr(), val, order) } + } + + /// Minimum with the current value. + /// + /// Finds the minimum of the current value and the argument `val`, and + /// sets the new value to the result. + /// + /// Returns the previous value. + /// + /// `fetch_min` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. All ordering modes are possible. Note that using + /// [`Acquire`] makes the store part of this operation [`Relaxed`], and + /// using [`Release`] makes the load part [`Relaxed`]. + /// + /// **Note**: This method is only available on platforms that support atomic operations on + #[doc = concat!("[`", $s_int_type, "`].")] + /// + /// # Examples + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// assert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 23); + /// assert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23); + /// assert_eq!(foo.load(Ordering::Relaxed), 22); + /// ``` + /// + /// If you want to obtain the minimum value in one step, you can use the following: + /// + #[cfg_attr($cfg_cas, doc = "```")] + #[cfg_attr(not($cfg_cas), doc = "```compile_fail")] + #[doc = concat!($extra_feature, "use std::sync::atomic::{", stringify!($atomic_type), ", Ordering};")] + /// + #[doc = concat!("let foo = ", stringify!($atomic_type), "::new(23);")] + /// let bar = 12; + /// let min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar); + /// assert_eq!(min_foo, 12); + /// ``` + #[inline] + #[stable(feature = "atomic_min_max", since = "1.45.0")] + #[rustc_const_unstable(feature = "const_atomic", issue = "160078")] + #[cfg(any($cfg_cas, doc))] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { + // SAFETY: data races are prevented by atomic intrinsics. + unsafe { $min_fn(self.as_ptr(), val, order) } + } + + /// Returns a mutable pointer to the underlying integer. + /// + /// Doing non-atomic reads and writes on the resulting integer can be a data race. + /// This method is mostly useful for FFI, where the function signature may use + #[doc = concat!("`*mut ", stringify!($int_type), "` instead of `&", stringify!($atomic_type), "`.")] + /// + /// Returning an `*mut` pointer from a shared reference to this atomic is safe because the + /// atomic types work with interior mutability. All modifications of an atomic change the value + /// through a shared reference, and can do so safely as long as they use atomic operations. Any + /// use of the returned raw pointer requires an `unsafe` block and still has to uphold the + /// requirements of the [memory model]. + /// + /// # Examples + /// + /// ```ignore (extern-declaration) + /// # fn main() { + #[doc = concat!($extra_feature, "use std::sync::atomic::", stringify!($atomic_type), ";")] + /// + /// extern "C" { + #[doc = concat!(" fn my_atomic_op(arg: *mut ", stringify!($int_type), ");")] + /// } + /// + #[doc = concat!("let atomic = ", stringify!($atomic_type), "::new(1);")] + /// + /// // SAFETY: Safe as long as `my_atomic_op` is atomic. + /// unsafe { + /// my_atomic_op(atomic.as_ptr()); + /// } + /// # } + /// ``` + /// + /// [memory model]: self#memory-model-for-atomic-accesses + #[inline] + #[stable(feature = "atomic_as_ptr", since = "1.70.0")] + #[rustc_const_stable(feature = "atomic_as_ptr", since = "1.70.0")] + #[rustc_never_returns_null_ptr] + pub const fn as_ptr(&self) -> *mut $int_type { + self.v.get().cast() + } + } + } +} + #[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { "i128", i128, AtomicI128 } +atomic_int_legacy! { + target_has_atomic_load_store = "128", + target_has_atomic = "128", + target_has_atomic_primitive_alignment = "128", + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + rustc_const_unstable(feature = "integer_atomics", issue = "99069"), + rustc_const_unstable(feature = "integer_atomics", issue = "99069"), + "i128", + "#![feature(integer_atomics)]\n\n", + atomic_min, atomic_max, + 16, + i128 AtomicI128 +} + #[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { "u128", u128, AtomicU128 } +atomic_int_legacy! { + target_has_atomic_load_store = "128", + target_has_atomic = "128", + target_has_atomic_primitive_alignment = "128", + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + unstable(feature = "integer_atomics", issue = "99069"), + rustc_const_unstable(feature = "integer_atomics", issue = "99069"), + rustc_const_unstable(feature = "integer_atomics", issue = "99069"), + "u128", + "#![feature(integer_atomics)]\n\n", + atomic_umin, atomic_umax, + 16, + u128 AtomicU128 +} #[cfg(target_has_atomic_load_store = "ptr")] macro_rules! atomic_int_ptr_sized { From 034769c672b1c0b0f4cfa8483db95c41749b3c4c Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 01:53:32 -0400 Subject: [PATCH 04/16] add one more trait to represent load_store support, allow impl macro to have no traits passed to it --- library/core/src/sync/atomic.rs | 89 ++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index fba1e609f9865..55456a8fa35d0 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -290,6 +290,26 @@ mod private { /// `Self` reference/pointer to `Self::Storage`. Transmuting a `Self` reference/pointer to /// `Self::Storage` can only be done if the reference/pointer has the same alignment as /// `Self::Storage`. +#[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" +)] +pub impl(self) unsafe trait AtomicPrimitive: Sized + Copy { + /// Temporary implementation detail. + type Storage: Sized; +} + +/// A marker trait for primitive types which can perform load/store operations atomically. +/// +/// This is an implementation detail for [Atomic]\ which may disappear or be +/// replaced at any time. +/// +/// # Safety +/// +/// Types implementing this trait must be primitives that can perform load/store operations +/// atomically. +/// /// /// The associated `Self::OpType` type must be an integer or pointer type that is the same size as /// `Self`. This is the type used with the atomic_load and atomic_store functions. Transmuting @@ -311,9 +331,7 @@ mod private { reason = "implementation detail which may disappear or be replaced at any time", issue = "none" )] -pub impl(self) unsafe trait AtomicPrimitive: Sized + Copy { - /// Temporary implementation detail. - type Storage: Sized; +pub impl(self) unsafe trait AtomicLoadStore: Sized + Copy + AtomicPrimitive { /// Temporary implementation detail. type OpType: Sized + Copy; } @@ -342,7 +360,7 @@ pub impl(self) unsafe trait AtomicPrimitive: Sized + Copy { reason = "implementation detail which may disappear or be replaced at any time", issue = "none" )] -pub impl(self) unsafe trait AtomicCas: AtomicPrimitive {} +pub impl(self) unsafe trait AtomicCas: AtomicLoadStore {} /// A market trait for primitive types which have the same alignment as their atomic counter-parts. /// @@ -362,7 +380,7 @@ pub impl(self) unsafe trait AtomicCas: AtomicPrimitive {} reason = "implementation detail which may disappear or be replaced at any time", issue = "none" )] -pub impl(self) unsafe trait AtomicAlignedPrimitive: AtomicPrimitive {} +pub impl(self) unsafe trait AtomicAlignedPrimitive: AtomicLoadStore {} /// A marker trait for atomic integer types. /// @@ -429,8 +447,7 @@ macro_rules! impl_atomic_traits { issue = "none" )] #[cfg(any(target_has_atomic_load_store = $size, doc))] - unsafe impl $(<$T>)? AtomicPrimitive for $Primitive { - type Storage = private::$Storage<$Operand>; + unsafe impl $(<$T>)? AtomicLoadStore for $Primitive { type OpType = $Operand; } }; @@ -511,20 +528,40 @@ macro_rules! impl_atomic_traits { ( [$T:ident] $Primitive:ty as $Storage:ident<$Operand:ty>, - size($size:literal), $($trait:ident),+ $(,)? + size($size:literal) $(, $trait:ident)* ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic_load_store = $size, doc))] + unsafe impl <$T> AtomicPrimitive for $Primitive { + type Storage = private::$Storage<$Operand>; + } + $( impl_atomic_traits!(@impl [$T] $Primitive as $Storage<$Operand>, size($size), $trait); - )+ + )* }; ( $Primitive:ty as $Storage:ident<$Operand:ty>, - size($size:literal), $($trait:ident),+ $(,)? + size($size:literal) $(, $trait:ident)* ) => { + #[unstable( + feature = "atomic_internals", + reason = "implementation detail which may disappear or be replaced at any time", + issue = "none" + )] + #[cfg(any(target_has_atomic_load_store = $size, doc))] + unsafe impl AtomicPrimitive for $Primitive { + type Storage = private::$Storage<$Operand>; + } + $( impl_atomic_traits!(@impl [] $Primitive as $Storage<$Operand>, size($size), $trait); - )+ + )* }; } @@ -538,28 +575,8 @@ impl_atomic_traits!(i32 as Align4, size("32"), load_store, cas, aligned, si impl_atomic_traits!(u32 as Align4, size("32"), load_store, cas, aligned, unsigned, bitwise); impl_atomic_traits!(i64 as Align8, size("64"), load_store, cas, aligned, signed, bitwise); impl_atomic_traits!(u64 as Align8, size("64"), load_store, cas, aligned, unsigned, bitwise); - -#[unstable( - feature = "atomic_internals", - reason = "implementation detail which may disappear or be replaced at any time", - issue = "none" -)] -#[cfg(any(target_has_atomic_load_store = "128", doc))] -unsafe impl AtomicPrimitive for i128 { - type Storage = private::Align16; - type OpType = i128; -} - -#[unstable( - feature = "atomic_internals", - reason = "implementation detail which may disappear or be replaced at any time", - issue = "none" -)] -#[cfg(any(target_has_atomic_load_store = "128", doc))] -unsafe impl AtomicPrimitive for u128 { - type Storage = private::Align16; - type OpType = u128; -} +impl_atomic_traits!(u128 as Align16, size("128")); +impl_atomic_traits!(i128 as Align16, size("128")); #[cfg(target_pointer_width = "16")] impl_atomic_traits!(isize as Align2, size("ptr"), load_store, cas, aligned, signed, bitwise); @@ -748,7 +765,7 @@ pub enum Ordering { SeqCst, } -impl Atomic { +impl Atomic { /// Creates a new `Atomic` with the given value. /// /// # Examples @@ -1860,7 +1877,7 @@ impl Atomic { } #[stable(feature = "integer_atomics_stable", since = "1.34.0")] -impl Default for Atomic { +impl Default for Atomic { #[inline] fn default() -> Self { Self::new(T::default()) @@ -1869,7 +1886,7 @@ impl Default for Atomic { #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] -impl From for Atomic { +impl From for Atomic { fn from(value: T) -> Self { Self::new(value) } From d44debccc392198de243c9611b1c54ad1bfdc1b6 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 02:06:25 -0400 Subject: [PATCH 05/16] make atomic_int_legacy not error out when 128 bit atomics aren't available --- library/core/src/sync/atomic.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 55456a8fa35d0..6d01842d73d8e 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -2966,7 +2966,7 @@ atomic_int! { "i64", i64, AtomicI64 } #[cfg(target_has_atomic_load_store = "64")] atomic_int! { "u64", u64, AtomicU64 } -#[cfg(target_has_atomic_load_store)] +#[cfg(target_has_atomic_load_store = "128")] macro_rules! atomic_int_legacy { ($cfg_base:meta, $cfg_cas:meta, @@ -4212,7 +4212,6 @@ atomic_int_legacy! { 16, i128 AtomicI128 } - #[cfg(any(target_has_atomic_load_store = "128", doc))] atomic_int_legacy! { target_has_atomic_load_store = "128", From 33c7762700a2e94553b8c26c53135c0ea36b27f0 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 02:29:30 -0400 Subject: [PATCH 06/16] fix from_ptr_raw signature to return pointer to Self instead of T --- library/core/src/sync/atomic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 6d01842d73d8e..88689bc288c61 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -902,7 +902,7 @@ impl Atomic { /// a reference to the destination. #[inline] #[unstable(feature = "atomic_volatile", issue = "158947")] - pub const fn from_ptr_raw(ptr: *mut T) -> *const T { + pub const fn from_ptr_raw(ptr: *mut T) -> *const Self { ptr.cast_const().cast() } From 09d59a02cab3f4c6b9aeaf05e502e76e04d44954 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 02:49:22 -0400 Subject: [PATCH 07/16] satisfy rustfmt --- library/core/src/sync/atomic.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 88689bc288c61..76c0aa1d2d017 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -331,7 +331,9 @@ pub impl(self) unsafe trait AtomicPrimitive: Sized + Copy { reason = "implementation detail which may disappear or be replaced at any time", issue = "none" )] -pub impl(self) unsafe trait AtomicLoadStore: Sized + Copy + AtomicPrimitive { +pub impl(self) unsafe trait AtomicLoadStore: + Sized + Copy + AtomicPrimitive +{ /// Temporary implementation detail. type OpType: Sized + Copy; } From 56e108ee2372ca0d1f94c7a5b6c3645a1d072238 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 09:20:52 -0400 Subject: [PATCH 08/16] add clippy markers to legacy const values with interior mutability --- library/core/src/sync/atomic.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 76c0aa1d2d017..0dc37baae9d6d 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -4250,6 +4250,7 @@ macro_rules! atomic_int_ptr_sized { note = "the `new` function is now preferred", suggestion = "AtomicIsize::new(0)", )] + #[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] pub const ATOMIC_ISIZE_INIT: AtomicIsize = AtomicIsize::new(0); /// An [`AtomicUsize`] initialized to `0`. @@ -4260,6 +4261,7 @@ macro_rules! atomic_int_ptr_sized { note = "the `new` function is now preferred", suggestion = "AtomicUsize::new(0)", )] + #[expect(clippy::declare_interior_mutable_const, reason = "legacy atomic initializer")] pub const ATOMIC_USIZE_INIT: AtomicUsize = AtomicUsize::new(0); )* }; } From 3dfd620ed19e48f5d26dab6b0ccef1308ff34f13 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 09:22:21 -0400 Subject: [PATCH 09/16] add semicolon to make rustfmt happy --- library/core/src/sync/atomic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 0dc37baae9d6d..142497e0a5f0f 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -2948,7 +2948,7 @@ macro_rules! atomic_int { fmt::Debug::fmt(&self.load(Ordering::Relaxed), f) } } - } + }; } #[cfg(target_has_atomic_load_store = "8")] From bcccb9c1e05df305123158dd77e7cb6497cdfb2e Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 10:23:43 -0400 Subject: [PATCH 10/16] make rustfmt happy because somehow this is what it wants --- library/core/src/sync/atomic.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 142497e0a5f0f..aef5c1b265516 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -2908,27 +2908,27 @@ macro_rules! if_8_bit { #[cfg(target_has_atomic_load_store)] macro_rules! atomic_int { ( - $s_int_type:literal, $int_type:ident, $atomic_type:ident + $s_int_type:literal, $int_type:ident, $atomic_type:ident ) => { /// An integer type which can be safely shared between threads. /// /// This type has the same #[doc = if_8_bit!( - $int_type, - yes = ["size, alignment, and bit validity"], - no = ["size and bit validity"], - )] + $int_type, + yes = ["size, alignment, and bit validity"], + no = ["size and bit validity"], + )] /// as the underlying integer type, [` #[doc = $s_int_type] /// `]. #[doc = if_8_bit! { - $int_type, - no = [ - "However, the alignment of this type is always equal to its ", - "size, even on targets where [`", $s_int_type, "`] has a ", - "lesser alignment." - ], - }] + $int_type, + no = [ + "However, the alignment of this type is always equal to its ", + "size, even on targets where [`", $s_int_type, "`] has a ", + "lesser alignment." + ], + }] /// For more about the differences between atomic types and /// non-atomic types as well as information about the portability of /// this type, please see the [module-level documentation]. From fe6ffc072ceeea98f48cd644156450eca6d5562a Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 10:25:05 -0400 Subject: [PATCH 11/16] duplicate safety comment directly over each unsafe call to make tidy happy --- library/core/src/sync/atomic.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index aef5c1b265516..48110f725c62d 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -1695,10 +1695,11 @@ impl Atomic { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub const fn fetch_max(&self, val: T, order: Ordering) -> T { - // SAFETY: data races are prevented by atomic intrinsics. if T::IS_SIGNED { + // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_max(self.as_ptr(), val, order) } } else { + // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_umax(self.as_ptr(), val, order) } } } @@ -1743,10 +1744,11 @@ impl Atomic { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[rustc_should_not_be_called_on_const_items] pub const fn fetch_min(&self, val: T, order: Ordering) -> T { - // SAFETY: data races are prevented by atomic intrinsics. if ::IS_SIGNED { + // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_min(self.as_ptr(), val, order) } } else { + // SAFETY: data races are prevented by atomic intrinsics. unsafe { atomic_umin(self.as_ptr(), val, order) } } } From ff4789b1ce83f945f0608bff0f903e42602db358 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 10:42:25 -0400 Subject: [PATCH 12/16] fix typo --- library/core/src/sync/atomic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 48110f725c62d..ac15adf88682d 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -811,7 +811,7 @@ impl Atomic { unsafe { // We can't use UnsafeCell's into_inner despite it being "older" (present since 1.0.0) // because it's const variant wasn't stabilized until 1.83.0 and the const variant of - // this function was stablizied earlier in 1.79.0 + // this function was stabilized earlier in 1.79.0 transmute_unchecked(self) } } From c3f06ccc8c3e750d227d584cec6df096cf2c90a6 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 11:50:21 -0400 Subject: [PATCH 13/16] match atomic_int_legacy cfg to 128 bit atomic macro calls cfg --- library/core/src/sync/atomic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index ac15adf88682d..d06e8ef7abf96 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -2970,7 +2970,7 @@ atomic_int! { "i64", i64, AtomicI64 } #[cfg(target_has_atomic_load_store = "64")] atomic_int! { "u64", u64, AtomicU64 } -#[cfg(target_has_atomic_load_store = "128")] +#[cfg(any(target_has_atomic_load_store = "128", doc))] macro_rules! atomic_int_legacy { ($cfg_base:meta, $cfg_cas:meta, From 9df609929c3012d7d344a187bf45a23c103f013c Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 14:09:57 -0400 Subject: [PATCH 14/16] rebless const_refers_to_static.rs tests due to change of where atomic fetch_add is implemented --- tests/ui/consts/miri_unleashed/const_refers_to_static.stderr | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr b/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr index e718aac5329a4..d247ee01f0a34 100644 --- a/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr +++ b/tests/ui/consts/miri_unleashed/const_refers_to_static.stderr @@ -6,12 +6,8 @@ LL | FOO.fetch_add(1, Ordering::Relaxed) | note: inside `Atomic::::fetch_add` --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL - ::: $SRC_DIR/core/src/sync/atomic.rs:LL:COL - | - = note: in this macro invocation note: inside `atomic::atomic_add::` --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL - = note: this error originates in the macro `atomic_int` which comes from the expansion of the macro `atomic_int_ptr_sized` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0080]: constant accesses mutable global memory --> $DIR/const_refers_to_static.rs:16:14 From 89f0505631fe72d115f9010170811b86b0ff6888 Mon Sep 17 00:00:00 2001 From: Nahla Date: Wed, 2 Sep 2026 14:51:38 -0400 Subject: [PATCH 15/16] fix docs to pass tests --- library/core/src/sync/atomic.rs | 114 ++++++++++++++++---------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index d06e8ef7abf96..b3df1cbe4c5ab 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -773,9 +773,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::Atomic; + /// use std::sync::atomic::AtomicBool; /// - /// let atomic = Atomic:::new(true); + /// let atomic = AtomicBool::new(true); /// ``` #[stable(feature = "integer_atomics_stable", since = "1.34.0")] #[rustc_const_stable(feature = "const_atomic_new", since = "1.34.0")] @@ -794,9 +794,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::Atomic; + /// use std::sync::atomic::AtomicBool; /// - /// let some_bool = Atomic::::new(true); + /// let some_bool = AtomicBool::new(true); /// assert_eq!(some_bool.into_inner(), true); /// ``` #[inline] @@ -830,13 +830,13 @@ impl Atomic { /// # Examples /// /// ```ignore (extern-declaration) - /// use std::sync::atomic::Atomic; + /// use std::sync::atomic::AtomicBool; /// /// extern "C" { /// fn my_atomic_op(arg: *mut bool); /// } /// - /// let mut atomic = Atomic::::new(true); + /// let mut atomic = AtomicBool::new(true); /// unsafe { /// my_atomic_op(atomic.as_ptr()); /// } @@ -857,16 +857,16 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// /// // Get a pointer to an allocated value /// let ptr: *mut bool = Box::into_raw(Box::new(false)); /// - /// assert!(ptr.cast::>().is_aligned()); + /// assert!(ptr.cast::().is_aligned()); /// /// { /// // Create an atomic view of the allocated value - /// let atomic = unsafe { Atomic::::from_ptr(ptr) }; + /// let atomic = unsafe { AtomicBool::from_ptr(ptr) }; /// /// // Use `atomic` for atomic operations, possibly share it with other threads /// atomic.store(true, Ordering::Relaxed); @@ -916,9 +916,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let mut some_bool = Atomic::::new(true); + /// let mut some_bool = AtomicBool::new(true); /// assert_eq!(*some_bool.get_mut(), true); /// *some_bool.get_mut() = false; /// assert_eq!(some_bool.load(Ordering::SeqCst), false); @@ -942,11 +942,11 @@ impl Atomic { /// # Examples /// /// ```rust,ignore-wasm - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let mut some_bools = [const { Atomic::::new(false) }; 10]; + /// let mut some_bools = [const { AtomicBool::new(false) }; 10]; /// - /// let view: &mut [bool] = Atomic::::get_mut_slice(&mut some_bools); + /// let view: &mut [bool] = AtomicBool::get_mut_slice(&mut some_bools); /// assert_eq!(view, [false; 10]); /// view[..5].copy_from_slice(&[true; 5]); /// @@ -980,9 +980,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let some_bool = Atomic::::new(true); + /// let some_bool = AtomicBool::new(true); /// /// assert_eq!(some_bool.load(Ordering::Relaxed), true); /// ``` @@ -1032,11 +1032,11 @@ impl Atomic { /// /// ```rust,no_run /// #![feature(atomic_volatile)] - /// use std::sync::atomic::{fence, Atomic, Ordering}; + /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; /// use std::ptr; /// /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); - /// let atomic_ptr = Atomic::<*mut u8>::from_ptr_raw(MMIO_ADDR); + /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); /// /// // Spin until we see a non-zero value. /// let buf = 'buf: loop { @@ -1077,9 +1077,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// - /// let some_bool = Atomic::::new(true); + /// let some_bool = AtomicBool::new(true); /// /// some_bool.store(false, Ordering::Relaxed); /// assert_eq!(some_bool.load(Ordering::Relaxed), false); @@ -1130,11 +1130,11 @@ impl Atomic { /// /// ```rust,no_run /// #![feature(atomic_volatile)] - /// use std::sync::atomic::{fence, Atomic, Ordering}; + /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; /// use std::ptr; /// /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); - /// let atomic_ptr = Atomic::<*mut u8>::from_ptr_raw(MMIO_ADDR); + /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); /// /// // Prepare some data for the DMA device. /// # fn get_dma_buffer() -> *mut u8 { panic!() } @@ -1175,9 +1175,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let some_atomic = Atomic::::new(5); + /// let some_atomic = AtomicU32::new(5); /// let value = some_atomic.swap(10, Ordering::Relaxed); /// ``` #[inline] @@ -1228,9 +1228,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let some_atomic = Atomic::::new(5); + /// let some_atomic = AtomicU32::new(5); /// /// let value = some_atomic.compare_and_swap(5, 10, Ordering::Relaxed); /// ``` @@ -1324,9 +1324,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let some_atomic = Atomic::::new(5); + /// let some_atomic = AtomicU32::new(5); /// /// let mut old = some_atomic.load(Ordering::Relaxed); /// loop { @@ -1423,9 +1423,9 @@ impl Atomic { /// # Examples /// /// ```rust - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let some_atomic = Atomic::::new(5); + /// let some_atomic = AtomicU32::new(5); /// /// assert_eq!(some_atomic.try_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(5)); /// let result = some_atomic.try_update(Ordering::SeqCst, Ordering::SeqCst, |x| { @@ -1495,12 +1495,12 @@ impl Atomic { /// /// ```rust /// - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let some_atomic = Atomic::new(5); + /// let some_atomic = AtomicU32::new(5); /// /// let result = some_atomic.update(Ordering::SeqCst, Ordering::SeqCst, |_| 10); - /// assert_eq!(result, ptr); + /// assert_eq!(result, 5); /// assert_eq!(some_atomic.load(Ordering::SeqCst), 10); /// ``` #[inline] @@ -1532,10 +1532,10 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// /// let mut some_bool = true; - /// let a = Atomic::::from_mut(&mut some_bool); + /// let a = AtomicBool::from_mut(&mut some_bool); /// a.store(false, Ordering::Relaxed); /// assert_eq!(some_bool, false); /// ``` @@ -1569,10 +1569,10 @@ impl Atomic { /// # Examples /// /// ```rust,ignore-wasm - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicBool, Ordering}; /// /// let mut some_bools = [false; 10]; - /// let a = &*Atomic::::from_mut_slice(&mut some_bools); + /// let a = &*AtomicBool::from_mut_slice(&mut some_bools); /// std::thread::scope(|s| { /// for i in 0..a.len() { /// s.spawn(move || a[i].store(true, Ordering::Relaxed)); @@ -1613,9 +1613,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(0); + /// let foo = AtomicU32::new(0); /// assert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0); /// assert_eq!(foo.load(Ordering::SeqCst), 10); /// ``` @@ -1641,9 +1641,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(20); + /// let foo = AtomicU32::new(20); /// assert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20); /// assert_eq!(foo.load(Ordering::SeqCst), 10); /// ``` @@ -1672,9 +1672,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(23); + /// let foo = AtomicU32::new(23); /// assert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23); /// assert_eq!(foo.load(Ordering::SeqCst), 42); /// ``` @@ -1682,9 +1682,9 @@ impl Atomic { /// If you want to obtain the maximum value in one step, you can use the following: /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(23); + /// let foo = AtomicU32::new(23); /// let bar = 42; /// let max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar); /// assert!(max_foo == 42); @@ -1719,9 +1719,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(23); + /// let foo = AtomicU32::new(23); /// assert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23); /// assert_eq!(foo.load(Ordering::Relaxed), 23); /// assert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23); @@ -1731,9 +1731,9 @@ impl Atomic { /// If you want to obtain the minimum value in one step, you can use the following: /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(23); + /// let foo = AtomicU32::new(23); /// let bar = 12; /// let min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar); /// assert_eq!(min_foo, 12); @@ -1770,9 +1770,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(0x13); + /// let foo = AtomicU32::new(0x13); /// assert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13); /// assert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31)); /// ``` @@ -1801,9 +1801,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(0b101101); + /// let foo = AtomicU32::new(0b101101); /// assert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101); /// assert_eq!(foo.load(Ordering::SeqCst), 0b100001); /// ``` @@ -1832,9 +1832,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(0b101101); + /// let foo = AtomicU32::new(0b101101); /// assert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101); /// assert_eq!(foo.load(Ordering::SeqCst), 0b111111); /// ``` @@ -1863,9 +1863,9 @@ impl Atomic { /// # Examples /// /// ``` - /// use std::sync::atomic::{Atomic, Ordering}; + /// use std::sync::atomic::{AtomicU32, Ordering}; /// - /// let foo = Atomic::::new(0b101101); + /// let foo = AtomicU32::new(0b101101); /// assert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); /// assert_eq!(foo.load(Ordering::SeqCst), 0b011110); /// ``` From 9a70f71f5f98edfb1a1537813964883d188f039a Mon Sep 17 00:00:00 2001 From: Nahla Date: Thu, 3 Sep 2026 13:16:54 -0400 Subject: [PATCH 16/16] bless test broken due to changes --- ...ut-not-available.alignment_mismatch.stderr | 23 ++++--------------- .../atomic-from-mut-not-available.rs | 2 +- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr index 4d17a6d4b5e9e..96486eb661d5e 100644 --- a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr +++ b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.alignment_mismatch.stderr @@ -1,26 +1,13 @@ -error[E0599]: no associated function or constant named `from_mut` found for struct `Atomic` in the current scope +error[E0599]: the associated function or constant `from_mut` exists for struct `Atomic`, but its trait bounds were not satisfied --> $DIR/atomic-from-mut-not-available.rs:25:36 | LL | core::sync::atomic::AtomicU64::from_mut(&mut 0u64); - | ^^^^^^^^ associated function or constant not found in `Atomic` + | ^^^^^^^^ associated function or constant cannot be called on `Atomic` due to unsatisfied trait bounds | -note: if you're trying to build a new `Atomic`, consider using `Atomic::::new` which returns `Atomic` +note: if you're trying to build a new `Atomic`, consider using `Atomic::::new` which returns `Atomic<_>` --> $SRC_DIR/core/src/sync/atomic.rs:LL:COL - ::: $SRC_DIR/core/src/sync/atomic.rs:LL:COL - | - = note: in this macro invocation - = note: the associated function or constant was found for - - `Atomic<*mut T>` - - `Atomic` - - `Atomic` - - `Atomic` - and 6 more types - = note: this error originates in the macro `atomic_int` (in Nightly builds, run with -Z macro-backtrace for more info) -help: there is an associated function `from` with a similar name - | -LL - core::sync::atomic::AtomicU64::from_mut(&mut 0u64); -LL + core::sync::atomic::AtomicU64::from(&mut 0u64); - | + = note: the following trait bounds were not satisfied: + `u64: AtomicAlignedPrimitive` error: aborting due to 1 previous error diff --git a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs index 0d5e09e6e2883..9f9f7e398a00b 100644 --- a/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs +++ b/tests/ui/stdlib-unit-tests/atomic-from-mut-not-available.rs @@ -23,5 +23,5 @@ fn main() { core::sync::atomic::AtomicU64::from_mut(&mut 0u64); - //[alignment_mismatch]~^ ERROR no associated function or constant named `from_mut` found for struct `Atomic` + //[alignment_mismatch]~^ERROR: the associated function or constant `from_mut` exists for struct `Atomic`, but its trait bounds were not satisfied [E0599] }