From 16bd406d43c28442e7d9f895c6cf7baae1eda0b5 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Mon, 14 Aug 2023 07:11:50 +0900 Subject: [PATCH] AtomicCell: Use atomic-maybe-uninit --- .github/workflows/ci.yml | 6 +- Cargo.toml | 2 +- README.md | 4 +- crossbeam-channel/Cargo.toml | 2 +- crossbeam-channel/README.md | 4 +- crossbeam-channel/src/lib.rs | 1 + crossbeam-deque/Cargo.toml | 2 +- crossbeam-deque/README.md | 4 +- crossbeam-epoch/Cargo.toml | 2 +- crossbeam-epoch/README.md | 4 +- crossbeam-queue/Cargo.toml | 2 +- crossbeam-queue/README.md | 4 +- crossbeam-skiplist/Cargo.toml | 2 +- crossbeam-skiplist/README.md | 4 +- crossbeam-utils/Cargo.toml | 3 +- crossbeam-utils/README.md | 4 +- crossbeam-utils/build.rs | 3 + crossbeam-utils/src/atomic/atomic_cell.rs | 673 +++++++++++++++++----- crossbeam-utils/tests/atomic_cell.rs | 13 +- 19 files changed, 572 insertions(+), 167 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb3dbc81..8aa69dbb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,9 +40,9 @@ jobs: matrix: # aarch64/x86_64 macOS and aarch64 Linux are tested on Cirrus CI include: - - rust: '1.38' + - rust: '1.59' os: ubuntu-latest - - rust: '1.38' + - rust: '1.59' os: windows-latest - rust: stable os: ubuntu-latest @@ -83,7 +83,7 @@ jobs: fail-fast: false matrix: rust: - - '1.38' + - '1.59' - nightly runs-on: ubuntu-latest steps: diff --git a/Cargo.toml b/Cargo.toml index 905c8ded6..2869376f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam" # - Create "crossbeam-X.Y.Z" git tag version = "0.8.2" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam" diff --git a/README.md b/README.md index e44df6b88..f4b9e8357 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam#license) https://crates.io/crates/crossbeam) [![Documentation](https://docs.rs/crossbeam/badge.svg)]( https://docs.rs/crossbeam) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -94,7 +94,7 @@ crossbeam = "0.8" Crossbeam supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## Contributing diff --git a/crossbeam-channel/Cargo.toml b/crossbeam-channel/Cargo.toml index a15528029..f20bb2c23 100644 --- a/crossbeam-channel/Cargo.toml +++ b/crossbeam-channel/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-channel" # - Create "crossbeam-channel-X.Y.Z" git tag version = "0.5.8" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-channel" diff --git a/crossbeam-channel/README.md b/crossbeam-channel/README.md index 76ffcf872..296d229e4 100644 --- a/crossbeam-channel/README.md +++ b/crossbeam-channel/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-channel#license) https://crates.io/crates/crossbeam-channel) [![Documentation](https://docs.rs/crossbeam-channel/badge.svg)]( https://docs.rs/crossbeam-channel) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -48,7 +48,7 @@ crossbeam-channel = "0.5" Crossbeam Channel supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-channel/src/lib.rs b/crossbeam-channel/src/lib.rs index cc1ef112f..c1495e850 100644 --- a/crossbeam-channel/src/lib.rs +++ b/crossbeam-channel/src/lib.rs @@ -335,6 +335,7 @@ unreachable_pub )] #![cfg_attr(not(feature = "std"), no_std)] +#![allow(clippy::match_like_matches_macro)] use cfg_if::cfg_if; diff --git a/crossbeam-deque/Cargo.toml b/crossbeam-deque/Cargo.toml index 805a7e005..24e6a547e 100644 --- a/crossbeam-deque/Cargo.toml +++ b/crossbeam-deque/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-deque" # - Create "crossbeam-deque-X.Y.Z" git tag version = "0.8.3" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque" diff --git a/crossbeam-deque/README.md b/crossbeam-deque/README.md index 23c8794c2..9a25dff20 100644 --- a/crossbeam-deque/README.md +++ b/crossbeam-deque/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque#license) https://crates.io/crates/crossbeam-deque) [![Documentation](https://docs.rs/crossbeam-deque/badge.svg)]( https://docs.rs/crossbeam-deque) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -28,7 +28,7 @@ crossbeam-deque = "0.8" Crossbeam Deque supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-epoch/Cargo.toml b/crossbeam-epoch/Cargo.toml index 4065564bf..1c2e4c65d 100644 --- a/crossbeam-epoch/Cargo.toml +++ b/crossbeam-epoch/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-epoch" # - Create "crossbeam-epoch-X.Y.Z" git tag version = "0.9.15" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch" diff --git a/crossbeam-epoch/README.md b/crossbeam-epoch/README.md index 2840ea792..0025e8a36 100644 --- a/crossbeam-epoch/README.md +++ b/crossbeam-epoch/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch#license) https://crates.io/crates/crossbeam-epoch) [![Documentation](https://docs.rs/crossbeam-epoch/badge.svg)]( https://docs.rs/crossbeam-epoch) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -35,7 +35,7 @@ crossbeam-epoch = "0.9" Crossbeam Epoch supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-queue/Cargo.toml b/crossbeam-queue/Cargo.toml index 72c6c7e2a..38e9f0a82 100644 --- a/crossbeam-queue/Cargo.toml +++ b/crossbeam-queue/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-queue" # - Create "crossbeam-queue-X.Y.Z" git tag version = "0.3.8" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-queue" diff --git a/crossbeam-queue/README.md b/crossbeam-queue/README.md index 238bcad32..0a176758c 100644 --- a/crossbeam-queue/README.md +++ b/crossbeam-queue/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-queue#license) https://crates.io/crates/crossbeam-queue) [![Documentation](https://docs.rs/crossbeam-queue/badge.svg)]( https://docs.rs/crossbeam-queue) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -36,7 +36,7 @@ crossbeam-queue = "0.3" Crossbeam Queue supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-skiplist/Cargo.toml b/crossbeam-skiplist/Cargo.toml index 76edc460e..8708529c8 100644 --- a/crossbeam-skiplist/Cargo.toml +++ b/crossbeam-skiplist/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-skiplist" # - Create "crossbeam-skiplist-X.Y.Z" git tag version = "0.1.1" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-skiplist" diff --git a/crossbeam-skiplist/README.md b/crossbeam-skiplist/README.md index 787ce87a5..0a08e633d 100644 --- a/crossbeam-skiplist/README.md +++ b/crossbeam-skiplist/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-skiplist#license https://crates.io/crates/crossbeam-skiplist) [![Documentation](https://docs.rs/crossbeam-skiplist/badge.svg)]( https://docs.rs/crossbeam-skiplist) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -34,7 +34,7 @@ crossbeam-skiplist = "0.1" Crossbeam Skiplist supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-utils/Cargo.toml b/crossbeam-utils/Cargo.toml index a110da24e..f89c81264 100644 --- a/crossbeam-utils/Cargo.toml +++ b/crossbeam-utils/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-utils" # - Create "crossbeam-utils-X.Y.Z" git tag version = "0.8.16" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils" @@ -23,6 +23,7 @@ std = [] [dependencies] cfg-if = "1" +atomic-maybe-uninit = { version = "0.2.21", git = "https://github.com/taiki-e/atomic-maybe-uninit.git", branch = "cfg" } # Enable the use of loom for concurrency testing. # diff --git a/crossbeam-utils/README.md b/crossbeam-utils/README.md index 5491c7213..b2c67de6a 100644 --- a/crossbeam-utils/README.md +++ b/crossbeam-utils/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils#license) https://crates.io/crates/crossbeam-utils) [![Documentation](https://docs.rs/crossbeam-utils/badge.svg)]( https://docs.rs/crossbeam-utils) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -55,7 +55,7 @@ crossbeam-utils = "0.8" Crossbeam Utils supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-utils/build.rs b/crossbeam-utils/build.rs index 39785f030..616f0ae62 100644 --- a/crossbeam-utils/build.rs +++ b/crossbeam-utils/build.rs @@ -64,4 +64,7 @@ fn main() { if sanitize.contains("thread") { println!("cargo:rustc-cfg=crossbeam_sanitize_thread"); } + if !sanitize.is_empty() { + println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback"); + } } diff --git a/crossbeam-utils/src/atomic/atomic_cell.rs b/crossbeam-utils/src/atomic/atomic_cell.rs index c60efbc94..3bf52ee03 100644 --- a/crossbeam-utils/src/atomic/atomic_cell.rs +++ b/crossbeam-utils/src/atomic/atomic_cell.rs @@ -1,7 +1,9 @@ // Necessary for implementing atomic methods for `AtomicUnit` #![allow(clippy::unit_arg)] -use crate::primitive::sync::atomic::{self, AtomicBool}; +use crate::primitive::sync::atomic; +#[cfg(not(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)))] +use crate::primitive::sync::atomic::AtomicBool; use core::cell::UnsafeCell; use core::cmp; use core::fmt; @@ -15,6 +17,12 @@ use std::panic::{RefUnwindSafe, UnwindSafe}; use super::seq_lock::SeqLock; +const FORCE_FALLBACK: bool = cfg!(any( + miri, + crossbeam_loom, + crossbeam_atomic_cell_force_fallback, +)); + /// A thread-safe mutable memory location. /// /// This type is equivalent to [`Cell`], except it can also be shared among multiple threads. @@ -103,6 +111,8 @@ impl AtomicCell { /// # Examples /// /// ``` + /// # // Always use fallback for now on environments that do not support inline assembly. + /// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; } /// use crossbeam_utils::atomic::AtomicCell; /// /// // This type is internally represented as `AtomicUsize` so we can just use atomic @@ -472,6 +482,233 @@ macro_rules! impl_arithmetic { } } }; + ($t:ty, fetch_update, $example:tt) => { + impl AtomicCell<$t> { + /// Increments the current value by `val` and returns the previous value. + /// + /// The addition wraps on overflow. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_add(3), 7); + /// assert_eq!(a.load(), 10); + /// ``` + #[inline] + pub fn fetch_add(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(old.wrapping_add(val))) + .unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_add(val); + old + } + } + + /// Decrements the current value by `val` and returns the previous value. + /// + /// The subtraction wraps on overflow. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_sub(3), 7); + /// assert_eq!(a.load(), 4); + /// ``` + #[inline] + pub fn fetch_sub(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(old.wrapping_sub(val))) + .unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_sub(val); + old + } + } + + /// Applies bitwise "and" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_and(3), 7); + /// assert_eq!(a.load(), 3); + /// ``` + #[inline] + pub fn fetch_and(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(old & val)).unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } + } + + /// Applies bitwise "nand" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_nand(3), 7); + /// assert_eq!(a.load(), !(7 & 3)); + /// ``` + #[inline] + pub fn fetch_nand(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(!(old & val))).unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } + } + + /// Applies bitwise "or" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_or(16), 7); + /// assert_eq!(a.load(), 23); + /// ``` + #[inline] + pub fn fetch_or(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(old | val)).unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } + } + + /// Applies bitwise "xor" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_xor(2), 7); + /// assert_eq!(a.load(), 5); + /// ``` + #[inline] + pub fn fetch_xor(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(old ^ val)).unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } + } + + /// Compares and sets the maximum of the current value and `val`, + /// and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_max(2), 7); + /// assert_eq!(a.load(), 7); + /// ``` + #[inline] + pub fn fetch_max(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(cmp::max(old, val))).unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::max(old, val); + old + } + } + + /// Compares and sets the minimum of the current value and `val`, + /// and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + #[doc = $example] + /// + /// assert_eq!(a.fetch_min(2), 7); + /// assert_eq!(a.load(), 2); + /// ``` + #[inline] + pub fn fetch_min(&self, val: $t) -> $t { + if !FORCE_FALLBACK + && can_transmute::<$t, atomic_maybe_uninit::AtomicMaybeUninit<$t>>() + { + self.fetch_update(|old| Some(cmp::min(old, val))).unwrap() + } else { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::min(old, val); + old + } + } + } + }; ($t:ty, $atomic:ty, $example:tt) => { impl AtomicCell<$t> { /// Increments the current value by `val` and returns the previous value. @@ -490,7 +727,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_add(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { let a = unsafe { &*(self.as_ptr() as *const $atomic) }; a.fetch_add(val, Ordering::AcqRel) } else { @@ -518,7 +755,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_sub(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { let a = unsafe { &*(self.as_ptr() as *const $atomic) }; a.fetch_sub(val, Ordering::AcqRel) } else { @@ -544,7 +781,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_and(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { let a = unsafe { &*(self.as_ptr() as *const $atomic) }; a.fetch_and(val, Ordering::AcqRel) } else { @@ -570,7 +807,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_nand(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { let a = unsafe { &*(self.as_ptr() as *const $atomic) }; a.fetch_nand(val, Ordering::AcqRel) } else { @@ -596,7 +833,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_or(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { let a = unsafe { &*(self.as_ptr() as *const $atomic) }; a.fetch_or(val, Ordering::AcqRel) } else { @@ -622,7 +859,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_xor(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { let a = unsafe { &*(self.as_ptr() as *const $atomic) }; a.fetch_xor(val, Ordering::AcqRel) } else { @@ -649,7 +886,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_max(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { // TODO: Atomic*::fetch_max requires Rust 1.45. self.fetch_update(|old| Some(cmp::max(old, val))).unwrap() } else { @@ -676,7 +913,7 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_min(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { + if !FORCE_FALLBACK && can_transmute::<$t, $atomic>() { // TODO: Atomic*::fetch_min requires Rust 1.45. self.fetch_update(|old| Some(cmp::min(old, val))).unwrap() } else { @@ -691,124 +928,275 @@ macro_rules! impl_arithmetic { }; } -impl_arithmetic!(u8, atomic::AtomicU8, "let a = AtomicCell::new(7u8);"); -impl_arithmetic!(i8, atomic::AtomicI8, "let a = AtomicCell::new(7i8);"); -impl_arithmetic!(u16, atomic::AtomicU16, "let a = AtomicCell::new(7u16);"); -impl_arithmetic!(i16, atomic::AtomicI16, "let a = AtomicCell::new(7i16);"); -impl_arithmetic!(u32, atomic::AtomicU32, "let a = AtomicCell::new(7u32);"); -impl_arithmetic!(i32, atomic::AtomicI32, "let a = AtomicCell::new(7i32);"); -#[cfg(not(crossbeam_no_atomic_64))] -impl_arithmetic!(u64, atomic::AtomicU64, "let a = AtomicCell::new(7u64);"); -#[cfg(not(crossbeam_no_atomic_64))] -impl_arithmetic!(i64, atomic::AtomicI64, "let a = AtomicCell::new(7i64);"); -#[cfg(crossbeam_no_atomic_64)] -impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);"); -#[cfg(crossbeam_no_atomic_64)] -impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);"); -// TODO: AtomicU128 is unstable -// impl_arithmetic!(u128, atomic::AtomicU128, "let a = AtomicCell::new(7u128);"); -// impl_arithmetic!(i128, atomic::AtomicI128, "let a = AtomicCell::new(7i128);"); -impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);"); -impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);"); - -impl_arithmetic!( - usize, - atomic::AtomicUsize, - "let a = AtomicCell::new(7usize);" -); -impl_arithmetic!( - isize, - atomic::AtomicIsize, - "let a = AtomicCell::new(7isize);" -); - -impl AtomicCell { - /// Applies logical "and" to the current value and returns the previous value. - /// - /// # Examples - /// - /// ``` - /// use crossbeam_utils::atomic::AtomicCell; - /// - /// let a = AtomicCell::new(true); - /// - /// assert_eq!(a.fetch_and(true), true); - /// assert_eq!(a.load(), true); - /// - /// assert_eq!(a.fetch_and(false), true); - /// assert_eq!(a.load(), false); - /// ``` - #[inline] - pub fn fetch_and(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_and(val, Ordering::AcqRel) - } +atomic_maybe_uninit::cfg_has_atomic_8! { + impl_arithmetic!(u8, atomic::AtomicU8, "let a = AtomicCell::new(7u8);"); + impl_arithmetic!(i8, atomic::AtomicI8, "let a = AtomicCell::new(7i8);"); +} +atomic_maybe_uninit::cfg_no_atomic_8! { + impl_arithmetic!(u8, fallback, "let a = AtomicCell::new(7u8);"); + impl_arithmetic!(i8, fallback, "let a = AtomicCell::new(7i8);"); +} - /// Applies logical "nand" to the current value and returns the previous value. - /// - /// # Examples - /// - /// ``` - /// use crossbeam_utils::atomic::AtomicCell; - /// - /// let a = AtomicCell::new(true); - /// - /// assert_eq!(a.fetch_nand(false), true); - /// assert_eq!(a.load(), true); - /// - /// assert_eq!(a.fetch_nand(true), true); - /// assert_eq!(a.load(), false); - /// - /// assert_eq!(a.fetch_nand(false), false); - /// assert_eq!(a.load(), true); - /// ``` - #[inline] - pub fn fetch_nand(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_nand(val, Ordering::AcqRel) - } +atomic_maybe_uninit::cfg_has_atomic_16! { + impl_arithmetic!(u16, atomic::AtomicU16, "let a = AtomicCell::new(7u16);"); + impl_arithmetic!(i16, atomic::AtomicI16, "let a = AtomicCell::new(7i16);"); +} +atomic_maybe_uninit::cfg_no_atomic_16! { + impl_arithmetic!(u16, fallback, "let a = AtomicCell::new(7u16);"); + impl_arithmetic!(i16, fallback, "let a = AtomicCell::new(7i16);"); +} - /// Applies logical "or" to the current value and returns the previous value. - /// - /// # Examples - /// - /// ``` - /// use crossbeam_utils::atomic::AtomicCell; - /// - /// let a = AtomicCell::new(false); - /// - /// assert_eq!(a.fetch_or(false), false); - /// assert_eq!(a.load(), false); - /// - /// assert_eq!(a.fetch_or(true), false); - /// assert_eq!(a.load(), true); - /// ``` - #[inline] - pub fn fetch_or(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_or(val, Ordering::AcqRel) - } +atomic_maybe_uninit::cfg_has_atomic_32! { + impl_arithmetic!(u32, atomic::AtomicU32, "let a = AtomicCell::new(7u32);"); + impl_arithmetic!(i32, atomic::AtomicI32, "let a = AtomicCell::new(7i32);"); +} +atomic_maybe_uninit::cfg_no_atomic_32! { + impl_arithmetic!(u32, fallback, "let a = AtomicCell::new(7u32);"); + impl_arithmetic!(i32, fallback, "let a = AtomicCell::new(7i32);"); +} - /// Applies logical "xor" to the current value and returns the previous value. - /// - /// # Examples - /// - /// ``` - /// use crossbeam_utils::atomic::AtomicCell; - /// - /// let a = AtomicCell::new(true); - /// - /// assert_eq!(a.fetch_xor(false), true); - /// assert_eq!(a.load(), true); - /// - /// assert_eq!(a.fetch_xor(true), true); - /// assert_eq!(a.load(), false); - /// ``` - #[inline] - pub fn fetch_xor(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_xor(val, Ordering::AcqRel) - } +atomic_maybe_uninit::cfg_has_atomic_64! { + #[cfg(not(crossbeam_no_atomic_64))] + impl_arithmetic!(u64, atomic::AtomicU64, "let a = AtomicCell::new(7u64);"); + #[cfg(not(crossbeam_no_atomic_64))] + impl_arithmetic!(i64, atomic::AtomicI64, "let a = AtomicCell::new(7i64);"); + #[cfg(crossbeam_no_atomic_64)] + impl_arithmetic!(u64, fetch_update, "let a = AtomicCell::new(7u64);"); + #[cfg(crossbeam_no_atomic_64)] + impl_arithmetic!(i64, fetch_update, atomic::AtomicI64, "let a = AtomicCell::new(7i64);"); +} +atomic_maybe_uninit::cfg_no_atomic_64! { + impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);"); + impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);"); +} + +atomic_maybe_uninit::cfg_has_atomic_128! { + // TODO: AtomicU128 is unstable + // impl_arithmetic!(u128, atomic::AtomicU128, "let a = AtomicCell::new(7u128);"); + // impl_arithmetic!(i128, atomic::AtomicI128, "let a = AtomicCell::new(7i128);"); + impl_arithmetic!(u128, fetch_update, "let a = AtomicCell::new(7u128);"); + impl_arithmetic!(i128, fetch_update, "let a = AtomicCell::new(7i128);"); +} +atomic_maybe_uninit::cfg_no_atomic_128! { + impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);"); + impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);"); +} + +atomic_maybe_uninit::cfg_has_atomic_ptr! { + impl_arithmetic!(usize, atomic::AtomicUsize, "let a = AtomicCell::new(7usize);"); + impl_arithmetic!(isize, atomic::AtomicIsize, "let a = AtomicCell::new(7isize);"); +} +atomic_maybe_uninit::cfg_no_atomic_ptr! { + impl_arithmetic!(usize, fallback, "let a = AtomicCell::new(7usize);"); + impl_arithmetic!(isize, fallback, "let a = AtomicCell::new(7isize);"); +} + +// TODO +macro_rules! impl_bool_arithmetic { + (fallback) => { + impl AtomicCell { + /// Applies logical "and" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_and(true), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_and(false), true); + /// assert_eq!(a.load(), false); + /// ``` + #[inline] + pub fn fetch_and(&self, val: bool) -> bool { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } + + /// Applies logical "nand" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_nand(false), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_nand(true), true); + /// assert_eq!(a.load(), false); + /// + /// assert_eq!(a.fetch_nand(false), false); + /// assert_eq!(a.load(), true); + /// ``` + #[inline] + pub fn fetch_nand(&self, val: bool) -> bool { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } + + /// Applies logical "or" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(false); + /// + /// assert_eq!(a.fetch_or(false), false); + /// assert_eq!(a.load(), false); + /// + /// assert_eq!(a.fetch_or(true), false); + /// assert_eq!(a.load(), true); + /// ``` + #[inline] + pub fn fetch_or(&self, val: bool) -> bool { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } + + /// Applies logical "xor" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_xor(false), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_xor(true), true); + /// assert_eq!(a.load(), false); + /// ``` + #[inline] + pub fn fetch_xor(&self, val: bool) -> bool { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } + } + }; + () => { + impl AtomicCell { + /// Applies logical "and" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_and(true), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_and(false), true); + /// assert_eq!(a.load(), false); + /// ``` + #[inline] + pub fn fetch_and(&self, val: bool) -> bool { + let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; + a.fetch_and(val, Ordering::AcqRel) + } + + /// Applies logical "nand" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_nand(false), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_nand(true), true); + /// assert_eq!(a.load(), false); + /// + /// assert_eq!(a.fetch_nand(false), false); + /// assert_eq!(a.load(), true); + /// ``` + #[inline] + pub fn fetch_nand(&self, val: bool) -> bool { + let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; + a.fetch_nand(val, Ordering::AcqRel) + } + + /// Applies logical "or" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(false); + /// + /// assert_eq!(a.fetch_or(false), false); + /// assert_eq!(a.load(), false); + /// + /// assert_eq!(a.fetch_or(true), false); + /// assert_eq!(a.load(), true); + /// ``` + #[inline] + pub fn fetch_or(&self, val: bool) -> bool { + let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; + a.fetch_or(val, Ordering::AcqRel) + } + + /// Applies logical "xor" to the current value and returns the previous value. + /// + /// # Examples + /// + /// ``` + /// use crossbeam_utils::atomic::AtomicCell; + /// + /// let a = AtomicCell::new(true); + /// + /// assert_eq!(a.fetch_xor(false), true); + /// assert_eq!(a.load(), true); + /// + /// assert_eq!(a.fetch_xor(true), true); + /// assert_eq!(a.load(), false); + /// ``` + #[inline] + pub fn fetch_xor(&self, val: bool) -> bool { + let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; + a.fetch_xor(val, Ordering::AcqRel) + } + } + }; +} + +#[cfg(not(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)))] +atomic_maybe_uninit::cfg_has_atomic_8! { + impl_bool_arithmetic!(); +} +#[cfg(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback))] +atomic_maybe_uninit::cfg_has_atomic_8! { + impl_bool_arithmetic!(fallback); +} +atomic_maybe_uninit::cfg_no_atomic_8! { + impl_bool_arithmetic!(fallback); } impl Default for AtomicCell { @@ -925,13 +1313,25 @@ macro_rules! atomic { loop { atomic!(@check, $t, AtomicUnit, $a, $atomic_op); - atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op); - atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op); - atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op); - #[cfg(not(crossbeam_no_atomic_64))] - atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op); - // TODO: AtomicU128 is unstable - // atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op); + // Always use fallback for now on environments that do not support inline assembly. + #[cfg(not(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)))] + { + atomic_maybe_uninit::cfg_has_atomic_8! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_16! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_32! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_64! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_128! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + } break $fallback_op; } @@ -940,16 +1340,7 @@ macro_rules! atomic { /// Returns `true` if operations on `AtomicCell` are lock-free. const fn atomic_is_lock_free() -> bool { - // HACK(taiki-e): This is equivalent to `atomic! { T, _a, true, false }`, but can be used in const fn even in our MSRV (Rust 1.38). - let is_lock_free = can_transmute::() - | can_transmute::() - | can_transmute::() - | can_transmute::(); - #[cfg(not(crossbeam_no_atomic_64))] - let is_lock_free = is_lock_free | can_transmute::(); - // TODO: AtomicU128 is unstable - // let is_lock_free = is_lock_free | can_transmute::(); - is_lock_free + atomic! { T, _a, true, false } } /// Atomically reads data from `src`. diff --git a/crossbeam-utils/tests/atomic_cell.rs b/crossbeam-utils/tests/atomic_cell.rs index edb7a4bc0..4f376d34b 100644 --- a/crossbeam-utils/tests/atomic_cell.rs +++ b/crossbeam-utils/tests/atomic_cell.rs @@ -6,6 +6,15 @@ use crossbeam_utils::atomic::AtomicCell; #[test] fn is_lock_free() { + // Always use fallback for now on environments that do not support inline assembly. + if cfg!(any( + miri, + crossbeam_loom, + crossbeam_atomic_cell_force_fallback, + )) { + return; + } + struct UsizeWrap(usize); struct U8Wrap(bool); struct I16Wrap(i16); @@ -44,8 +53,8 @@ fn is_lock_free() { cfg!(not(crossbeam_no_atomic_64)) ); - // AtomicU128 is unstable - assert!(!AtomicCell::::is_lock_free()); + // // AtomicU128 is unstable + // assert!(!AtomicCell::::is_lock_free()); } #[test]