diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 9857a3b8cfae6..ba6d786fbbd89 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -108,6 +108,7 @@ #![feature(adt_const_params)] #![feature(allow_internal_unsafe)] #![feature(allow_internal_unstable)] +#![feature(arbitrary_self_types_pointers)] #![feature(auto_traits)] #![feature(cfg_sanitize)] #![feature(cfg_target_has_atomic)] diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index aedab87449ff5..f1b81de50942a 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -2067,12 +2067,13 @@ pub const unsafe fn write_unaligned(dst: *mut T, src: T) { /// are two cases of usage that need to be distinguished: /// /// - When a volatile operation is used for memory inside an [allocation], it behaves exactly like -/// [`read`], except for the additional guarantee that it won't be elided or reordered (see -/// above). This implies that the operation will actually access memory and not e.g. be lowered to -/// reusing data from a previous read. Other than that, all the usual rules for memory accesses -/// apply (including provenance). In particular, just like in C, whether an operation is volatile -/// has no bearing whatsoever on questions involving concurrent accesses from multiple threads. -/// Volatile accesses behave exactly like non-atomic accesses in that regard. +/// [`read`], except for the additional guarantee that it won't be elided or reordered across +/// other externally observable events (see above). This implies that the operation will actually +/// access memory and not e.g. be lowered to reusing data from a previous read. Other than that, +/// all the usual rules for memory accesses apply (including provenance). In particular, just +/// like in C, whether an operation is volatile has no bearing whatsoever on questions involving +/// concurrent accesses from multiple threads. Volatile accesses behave exactly like non-atomic +/// accesses in that regard. /// /// - Volatile operations, however, may also be used to access memory that is _outside_ of any Rust /// allocation. In this use-case, the pointer does *not* have to be [valid] for reads. This is @@ -2174,11 +2175,12 @@ pub const unsafe fn read_volatile(src: *const T) -> T { /// /// - When a volatile operation is used for memory inside an [allocation], it behaves exactly like /// [`write`][write()], except for the additional guarantee that it won't be elided or reordered -/// (see above). This implies that the operation will actually access memory and not e.g. be -/// lowered to a register access. Other than that, all the usual rules for memory accesses apply -/// (including provenance). In particular, just like in C, whether an operation is volatile has no -/// bearing whatsoever on questions involving concurrent access from multiple threads. Volatile -/// accesses behave exactly like non-atomic accesses in that regard. +/// across other externally observable events (see above). This implies that the operation will +/// actually access memory and not e.g. be lowered to a register access. Other than that, all the +/// usual rules for memory accesses apply (including provenance). In particular, just like in C, +/// whether an operation is volatile has no bearing whatsoever on questions involving concurrent +/// access from multiple threads. Volatile accesses behave exactly like non-atomic accesses in +/// that regard. /// /// - Volatile operations, however, may also be used to access memory that is _outside_ of any Rust /// allocation. In this use-case, the pointer does *not* have to be [valid] for writes. This is @@ -2187,7 +2189,7 @@ pub const unsafe fn read_volatile(src: *const T) -> T { /// semantics associated to their manipulation, and cannot be used as general purpose memory. /// Here, any address value is possible, including 0 and [`usize::MAX`], so long as the semantics /// of such a write are well-defined by the target hardware. The provenance of the pointer is -/// irrelevant, and it can be created with [`without_provenance`]. The access must not trap. It +/// irrelevant, and it can be created with [`without_provenance_mut`]. The access must not trap. It /// can cause side-effects, but those must not affect Rust-allocated memory in any way. This /// access is still not considered [atomic], and as such it cannot be used for inter-thread /// synchronization. diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 12208b95307ee..8b770528a1736 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -606,6 +606,16 @@ impl AtomicBool { unsafe { &*ptr.cast() } } + /// Creates a new pointer to `AtomicBool` from a pointer. + /// + /// This is useful if you want to do volatile atomic accesses, and thus avoid creating + /// a reference to the destination. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + pub const fn from_ptr_raw(ptr: *mut bool) -> *const AtomicBool { + ptr.cast_const().cast() + } + /// Returns a mutable reference to the underlying [`bool`]. /// /// This is safe because the mutable reference guarantees that no other threads are @@ -765,6 +775,40 @@ impl AtomicBool { } } + /// Perform a volatile atomic load from the bool. + /// + /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_load_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be [valid] for reads, or `self` must point to memory + /// outside of all Rust allocations and reading from that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// * Reading from `self` must produce a properly initialized value of type `bool`. + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Release`] or [`AcqRel`]. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> bool { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_load::<_, /* VOLATILE */ true>(self.cast::(), order) != 0 + } + } + /// Stores a value into the bool. /// /// `store` takes an [`Ordering`] argument which describes the memory ordering @@ -797,6 +841,39 @@ impl AtomicBool { } } + /// Performs a volatile atomic store into the bool. + /// + /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_store_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be either [valid] for writes, or `self` must point to memory + /// outside of all Rust allocations and writing to that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const unsafe fn store_volatile(self: *const Self, val: bool, order: Ordering) { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_store::<_, /* VOLATILE */ true>(self.cast::().cast_mut(), val as u8, order); + } + } + /// Stores a value into the bool, returning the previous value. /// /// `swap` takes an [`Ordering`] argument which describes the memory ordering @@ -1567,6 +1644,16 @@ impl AtomicPtr { unsafe { &*ptr.cast() } } + /// Creates a new pointer to `AtomicPtr` from a pointer. + /// + /// This is useful if you want to do volatile atomic accesses, and thus avoid creating + /// a reference to the destination. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + pub const fn from_ptr_raw(ptr: *mut *mut T) -> *const AtomicPtr { + ptr.cast_const().cast() + } + /// Creates a new `AtomicPtr` initialized with a null pointer. /// /// # Examples @@ -1770,6 +1857,71 @@ impl AtomicPtr { } } + /// Perform a volatile atomic load from the pointer. + /// + /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_load_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be [valid] for reads, or `self` must point to memory + /// outside of all Rust allocations and reading from that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// * `self` must be aligned to `align_of::>()` (note that on some platforms this + /// can be bigger than `align_of::<*mut T>()`). + /// + /// * Reading from `self` must produce a properly initialized value of type `*mut T`. + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Release`] or [`AcqRel`]. + /// + /// # Examples + /// + /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory + /// access, we may receive a buffer in shared memory from that device as follows: + /// + /// ```rust,no_run + /// #![feature(atomic_volatile)] + /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; + /// use std::ptr; + /// + /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); + /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); + /// + /// // Spin until we see a non-zero value. + /// let buf = 'buf: loop { + /// let buf = unsafe { atomic_ptr.load_volatile(Ordering::Relaxed) }; + /// if !buf.is_null() { + /// break 'buf buf; + /// } + /// }; + /// // Synchronize with the store whose value we just read. + /// // Note: a standard acquire fence may not be sufficient to synchronize with DMA devices. + /// // Depending on your target, you may have to use inline assembly to emit a special fence. + /// fence(Ordering::Acquire); + /// + /// // Now process the data in `buf`. + /// ``` + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> *mut T { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_load::<_, /* VOLATILE */ true>(self.cast::<*mut T>(), order) + } + } + /// Stores a value into the pointer. /// /// `store` takes an [`Ordering`] argument which describes the memory ordering @@ -1803,6 +1955,67 @@ impl AtomicPtr { } } + /// Performs a volatile atomic store into the pointer. + /// + /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_store_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be either [valid] for writes, or `self` must point to memory + /// outside of all Rust allocations and writing to that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// * `self` must be aligned to `align_of::>()` (note that on some platforms this + /// can be bigger than `align_of::<*mut T>()`). + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + /// + /// # Examples + /// + /// Assuming an MMIO region at `MMIO_ADDR` that belongs to a device with direct memory + /// access, we may submit a buffer in shared memory to that device as follows: + /// + /// ```rust,no_run + /// #![feature(atomic_volatile)] + /// use std::sync::atomic::{fence, AtomicPtr, Ordering}; + /// use std::ptr; + /// + /// const MMIO_ADDR: *mut *mut u8 = ptr::without_provenance_mut(0xCAF0); + /// let atomic_ptr = AtomicPtr::::from_ptr_raw(MMIO_ADDR); + /// + /// // Prepare some data for the DMA device. + /// # fn get_dma_buffer() -> *mut u8 { panic!() } + /// let buf = get_dma_buffer(); + /// + /// // Ensure the other side can synchronize with the store we do below. + /// // Note: a standard release fence may not be sufficient to synchronize with DMA devices. + /// // Depending on your target, you may have to use inline assembly to emit a special fence. + /// fence(Ordering::Release); + /// + /// unsafe { atomic_ptr.store_volatile(buf, Ordering::Relaxed) }; + /// ``` + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const unsafe fn store_volatile(self: *const Self, ptr: *mut T, order: Ordering) { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_store::<_, /* VOLATILE */ true>(self.cast::<*mut T>().cast_mut(), ptr, order); + } + } + /// Stores a value into the pointer, returning the previous value. /// /// `swap` takes an [`Ordering`] argument which describes the memory ordering @@ -2733,6 +2946,16 @@ macro_rules! atomic_int { unsafe { &*ptr.cast() } } + /// Creates a new pointer to an atomic integer from a pointer. + /// + /// This is useful if you want to do volatile atomic accesses, and thus avoid creating + /// a reference to the destination. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + pub const fn from_ptr_raw(ptr: *mut $int_type) -> *const $atomic_type { + ptr.cast_const().cast() + } + /// Returns a mutable reference to the underlying integer. /// /// This is safe because the mutable reference guarantees that no other threads are @@ -2921,6 +3144,55 @@ macro_rules! atomic_int { unsafe { atomic_load::<_, /* VOLATILE */ false>(self.as_ptr(), order) } } + /// Perform a volatile load from the atomic integer. + /// + /// `load_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_load_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be [valid] for reads, or `self` must point to memory + /// outside of all Rust allocations and reading from that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// * `self` must be aligned to + #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] + #[doc = if_8_bit!{ + $int_type, + yes = [ + " (note that this is always true, since `align_of::<", + stringify!($atomic_type), ">() == 1`)." + ], + no = [ + " (note that on some platforms this can be bigger than `align_of::<", + stringify!($int_type), ">()`)." + ], + }] + /// + /// * Reading from `self` must produce a properly initialized value of the underlying + /// integer type. + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Release`] or [`AcqRel`]. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub const unsafe fn load_volatile(self: *const Self, order: Ordering) -> $int_type { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_load::<_, /* VOLATILE */ true>(self.cast::<$int_type>(), order) + } + } + /// Stores a value into the atomic integer. /// /// `store` takes an [`Ordering`] argument which describes the memory ordering of this operation. @@ -2951,6 +3223,53 @@ macro_rules! atomic_int { unsafe { atomic_store::<_, /* VOLATILE */ false>(self.as_ptr(), val, order); } } + /// Performs a volatile store into the atomic integer. + /// + /// `store_volatile` takes an [`Ordering`] argument which describes the memory ordering + /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`]. + /// + #[doc = include_str!("./atomic_store_volatile.md")] + /// + /// # Safety + /// + /// Behavior is undefined if any of the following conditions are violated: + /// + /// * `self` must be either [valid] for writes, or `self` must point to memory + /// outside of all Rust allocations and writing to that memory must: + /// - not trap, and + /// - not cause any memory inside a Rust allocation to be modified. + /// + /// * `self` must be aligned to + #[doc = concat!(" `align_of::<", stringify!($atomic_type), ">()`")] + #[doc = if_8_bit!{ + $int_type, + yes = [ + " (note that this is always true, since `align_of::<", + stringify!($atomic_type), ">() == 1`)." + ], + no = [ + " (note that on some platforms this can be bigger than `align_of::<", + stringify!($int_type), ">()`)." + ], + }] + /// + /// [valid]: core::ptr#safety + /// + /// # Panics + /// + /// Panics if `order` is [`Acquire`] or [`AcqRel`]. + #[inline] + #[unstable(feature = "atomic_volatile", issue = "158947")] + #[rustc_const_unstable(feature = "atomic_volatile", issue = "158947")] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + #[rustc_should_not_be_called_on_const_items] + pub const unsafe fn store_volatile(self: *const Self, val: $int_type, order: Ordering) { + // SAFETY: follows from our own safety requirements. + unsafe { + atomic_store::<_, /* VOLATILE */ true>(self.cast::<$int_type>().cast_mut(), val, order); + } + } + /// Stores a value into the atomic integer, returning the previous value. /// /// `swap` takes an [`Ordering`] argument which describes the memory ordering diff --git a/library/core/src/sync/atomic_load_volatile.md b/library/core/src/sync/atomic_load_volatile.md new file mode 100644 index 0000000000000..be02c4bcfeccf --- /dev/null +++ b/library/core/src/sync/atomic_load_volatile.md @@ -0,0 +1,29 @@ +Volatile operations are intended to act on I/O memory. As such, they are considered externally +observable events (just like syscalls, but less opaque), and are guaranteed to not be elided or +reordered by the compiler across other externally observable events. With this in mind, there +are two cases of usage that need to be distinguished: + +- When a volatile operation is used for memory inside an [allocation], it behaves exactly + like [`load`][Self::load], except for the additional guarantee that it won't be elided or + reordered across other externally observable events (see above). This implies that the + operation will actually access memory and not e.g. be lowered to reusing data from a + previous load. Other than that, all the usual rules for memory accesses apply (including + provenance). + +- Volatile operations, however, may also be used to access memory that is _outside_ of any Rust + allocation. In this use-case, the pointer does *not* have to be [valid] for reads. This is + typically used for CPU and peripheral registers that must be accessed via an I/O memory mapping, + most commonly at fixed addresses reserved by the hardware. These often have special semantics + associated to their manipulation, and cannot be used as general purpose memory. Here, any address + value is possible, including 0 and [`usize::MAX`], so long as the semantics of such a read are + well-defined by the target hardware. The provenance of the pointer is irrelevant, and it can be + created with [`without_provenance`][crate::ptr::without_provenance]. The access must not trap. It + can cause side-effects, but those must not affect Rust-allocated memory in any way. + +In both cases, the access is also considered atomic with the given `order`. This allows +synchronization with other threads or devices that share memory with this program. + +When invoked during const evaluation, this behaves like a regular atomic load. In +particular, such reads must always follow the first of the two cases above. + +[allocation]: crate::ptr#allocated-object diff --git a/library/core/src/sync/atomic_store_volatile.md b/library/core/src/sync/atomic_store_volatile.md new file mode 100644 index 0000000000000..bb78099020c15 --- /dev/null +++ b/library/core/src/sync/atomic_store_volatile.md @@ -0,0 +1,28 @@ +Volatile operations are intended to act on I/O memory. As such, they are considered externally +observable events (just like syscalls), and are guaranteed to not be elided or reordered by the +compiler across other externally observable events. With this in mind, there are two cases of +usage that need to be distinguished: + +- When a volatile operation is used for memory inside an [allocation], it behaves exactly like + [`store`][Self::store], except for the additional guarantee that it won't be elided or reordered + across other externally observable events (see above). This implies that the operation will + actually access memory and not e.g. be lowered to a register access. Other than that, all the + usual rules for memory accesses apply (including provenance). + +- Volatile operations, however, may also be used to access memory that is _outside_ of any Rust + allocation. In this use-case, the pointer does *not* have to be [valid] for writes. This is + typically used for CPU and peripheral registers that must be accessed via an I/O memory mapping, + most commonly at fixed addresses reserved by the hardware. These often have special semantics + associated to their manipulation, and cannot be used as general purpose memory. Here, any address + value is possible, including 0 and [`usize::MAX`], so long as the semantics of such a write are + well-defined by the target hardware. The provenance of the pointer is irrelevant, and it can be + created with [`without_provenance_mut`][crate::ptr::without_provenance_mut]. The access must not + trap. It can cause side-effects, but those must not affect Rust-allocated memory in any way. + +In both cases, the access is also considered atomic with the given `order`. This allows +synchronization with other threads or devices that share memory with this program. + +When invoked during const evaluation, this behaves like a regular atomic store. In +particular, such reads must always follow the first of the two cases above. + +[allocation]: crate::ptr#allocated-object diff --git a/library/coretests/tests/atomic.rs b/library/coretests/tests/atomic.rs index d888bd0f55a11..31fe7980510d3 100644 --- a/library/coretests/tests/atomic.rs +++ b/library/coretests/tests/atomic.rs @@ -538,6 +538,29 @@ fn atomic_umin() { assert_eq!(ATOMIC.load(Relaxed), 0); } +#[test] +fn atomic_volatile() { + use Ordering::*; + + let mut b = true; + let atomic = AtomicBool::from_ptr_raw(&raw mut b); + assert!(unsafe { atomic.load_volatile(Relaxed) }); + unsafe { atomic.store_volatile(false, Relaxed) }; + assert!(!b); + + let mut ptr = std::ptr::null_mut::(); + let atomic = AtomicPtr::from_ptr_raw(&raw mut ptr); + assert!(unsafe { atomic.load_volatile(Relaxed) }.is_null()); + unsafe { atomic.store_volatile(std::ptr::without_provenance_mut(16), Relaxed) }; + assert!(ptr.addr() == 16); + + let mut int = 0i32; + let atomic = AtomicI32::from_ptr_raw(&raw mut int); + assert!(unsafe { atomic.load_volatile(Relaxed) } == 0); + unsafe { atomic.store_volatile(16, Relaxed) }; + assert!(int == 16); +} + /* FIXME(#110395) #[test] fn atomic_const_from() { diff --git a/library/coretests/tests/lib.rs b/library/coretests/tests/lib.rs index e81cae69e1852..142df37c2b7fe 100644 --- a/library/coretests/tests/lib.rs +++ b/library/coretests/tests/lib.rs @@ -8,6 +8,7 @@ #![feature(ascii_char_variants)] #![feature(async_iter_from_iter)] #![feature(async_iterator)] +#![feature(atomic_volatile)] #![feature(borrowed_buf_init)] #![feature(bstr)] #![feature(casefold)]