From f6c99b88e75845f58d5e0559c6874617501bf565 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Tue, 15 Aug 2023 20:01:09 +0900 Subject: [PATCH] AtomicCell: Use atomic-maybe-uninit --- .github/workflows/ci.yml | 6 +- Cargo.toml | 2 +- README.md | 4 +- crossbeam-channel/Cargo.toml | 2 +- crossbeam-channel/README.md | 4 +- crossbeam-channel/examples/stopwatch.rs | 2 +- crossbeam-channel/src/lib.rs | 1 + crossbeam-channel/src/select.rs | 6 +- crossbeam-deque/Cargo.toml | 2 +- crossbeam-deque/README.md | 4 +- crossbeam-deque/src/lib.rs | 1 + crossbeam-epoch/Cargo.toml | 2 +- crossbeam-epoch/README.md | 4 +- crossbeam-queue/Cargo.toml | 2 +- crossbeam-queue/README.md | 4 +- crossbeam-skiplist/Cargo.toml | 2 +- crossbeam-skiplist/README.md | 4 +- crossbeam-utils/Cargo.toml | 3 +- crossbeam-utils/README.md | 4 +- crossbeam-utils/build.rs | 3 + crossbeam-utils/src/atomic/atomic_cell.rs | 525 ++++++++++++++-------- crossbeam-utils/src/atomic/mod.rs | 16 +- crossbeam-utils/tests/atomic_cell.rs | 23 +- 23 files changed, 395 insertions(+), 231 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb3dbc81..8aa69dbb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,9 +40,9 @@ jobs: matrix: # aarch64/x86_64 macOS and aarch64 Linux are tested on Cirrus CI include: - - rust: '1.38' + - rust: '1.59' os: ubuntu-latest - - rust: '1.38' + - rust: '1.59' os: windows-latest - rust: stable os: ubuntu-latest @@ -83,7 +83,7 @@ jobs: fail-fast: false matrix: rust: - - '1.38' + - '1.59' - nightly runs-on: ubuntu-latest steps: diff --git a/Cargo.toml b/Cargo.toml index 905c8ded6..2869376f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam" # - Create "crossbeam-X.Y.Z" git tag version = "0.8.2" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam" diff --git a/README.md b/README.md index e44df6b88..f4b9e8357 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam#license) https://crates.io/crates/crossbeam) [![Documentation](https://docs.rs/crossbeam/badge.svg)]( https://docs.rs/crossbeam) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -94,7 +94,7 @@ crossbeam = "0.8" Crossbeam supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## Contributing diff --git a/crossbeam-channel/Cargo.toml b/crossbeam-channel/Cargo.toml index a15528029..f20bb2c23 100644 --- a/crossbeam-channel/Cargo.toml +++ b/crossbeam-channel/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-channel" # - Create "crossbeam-channel-X.Y.Z" git tag version = "0.5.8" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-channel" diff --git a/crossbeam-channel/README.md b/crossbeam-channel/README.md index 76ffcf872..296d229e4 100644 --- a/crossbeam-channel/README.md +++ b/crossbeam-channel/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-channel#license) https://crates.io/crates/crossbeam-channel) [![Documentation](https://docs.rs/crossbeam-channel/badge.svg)]( https://docs.rs/crossbeam-channel) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -48,7 +48,7 @@ crossbeam-channel = "0.5" Crossbeam Channel supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-channel/examples/stopwatch.rs b/crossbeam-channel/examples/stopwatch.rs index 3a7578e00..3a8962279 100644 --- a/crossbeam-channel/examples/stopwatch.rs +++ b/crossbeam-channel/examples/stopwatch.rs @@ -18,7 +18,7 @@ fn main() { // Creates a channel that gets a message every time `SIGINT` is signalled. fn sigint_notifier() -> io::Result> { let (s, r) = bounded(100); - let mut signals = Signals::new(&[SIGINT])?; + let mut signals = Signals::new([SIGINT])?; thread::spawn(move || { for _ in signals.forever() { diff --git a/crossbeam-channel/src/lib.rs b/crossbeam-channel/src/lib.rs index cc1ef112f..c1495e850 100644 --- a/crossbeam-channel/src/lib.rs +++ b/crossbeam-channel/src/lib.rs @@ -335,6 +335,7 @@ unreachable_pub )] #![cfg_attr(not(feature = "std"), no_std)] +#![allow(clippy::match_like_matches_macro)] use cfg_if::cfg_if; diff --git a/crossbeam-channel/src/select.rs b/crossbeam-channel/src/select.rs index 3eb0b97c8..f71069255 100644 --- a/crossbeam-channel/src/select.rs +++ b/crossbeam-channel/src/select.rs @@ -79,10 +79,10 @@ impl From for Selected { } } -impl Into for Selected { +impl From for usize { #[inline] - fn into(self) -> usize { - match self { + fn from(val: Selected) -> Self { + match val { Selected::Waiting => 0, Selected::Aborted => 1, Selected::Disconnected => 2, diff --git a/crossbeam-deque/Cargo.toml b/crossbeam-deque/Cargo.toml index 805a7e005..24e6a547e 100644 --- a/crossbeam-deque/Cargo.toml +++ b/crossbeam-deque/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-deque" # - Create "crossbeam-deque-X.Y.Z" git tag version = "0.8.3" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque" diff --git a/crossbeam-deque/README.md b/crossbeam-deque/README.md index 23c8794c2..9a25dff20 100644 --- a/crossbeam-deque/README.md +++ b/crossbeam-deque/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque#license) https://crates.io/crates/crossbeam-deque) [![Documentation](https://docs.rs/crossbeam-deque/badge.svg)]( https://docs.rs/crossbeam-deque) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -28,7 +28,7 @@ crossbeam-deque = "0.8" Crossbeam Deque supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-deque/src/lib.rs b/crossbeam-deque/src/lib.rs index 16bc728b3..096026d63 100644 --- a/crossbeam-deque/src/lib.rs +++ b/crossbeam-deque/src/lib.rs @@ -96,6 +96,7 @@ unreachable_pub )] #![cfg_attr(not(feature = "std"), no_std)] +#![allow(clippy::match_like_matches_macro)] use cfg_if::cfg_if; diff --git a/crossbeam-epoch/Cargo.toml b/crossbeam-epoch/Cargo.toml index 4065564bf..1c2e4c65d 100644 --- a/crossbeam-epoch/Cargo.toml +++ b/crossbeam-epoch/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-epoch" # - Create "crossbeam-epoch-X.Y.Z" git tag version = "0.9.15" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch" diff --git a/crossbeam-epoch/README.md b/crossbeam-epoch/README.md index 2840ea792..0025e8a36 100644 --- a/crossbeam-epoch/README.md +++ b/crossbeam-epoch/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch#license) https://crates.io/crates/crossbeam-epoch) [![Documentation](https://docs.rs/crossbeam-epoch/badge.svg)]( https://docs.rs/crossbeam-epoch) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -35,7 +35,7 @@ crossbeam-epoch = "0.9" Crossbeam Epoch supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-queue/Cargo.toml b/crossbeam-queue/Cargo.toml index 72c6c7e2a..38e9f0a82 100644 --- a/crossbeam-queue/Cargo.toml +++ b/crossbeam-queue/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-queue" # - Create "crossbeam-queue-X.Y.Z" git tag version = "0.3.8" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-queue" diff --git a/crossbeam-queue/README.md b/crossbeam-queue/README.md index 238bcad32..0a176758c 100644 --- a/crossbeam-queue/README.md +++ b/crossbeam-queue/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-queue#license) https://crates.io/crates/crossbeam-queue) [![Documentation](https://docs.rs/crossbeam-queue/badge.svg)]( https://docs.rs/crossbeam-queue) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -36,7 +36,7 @@ crossbeam-queue = "0.3" Crossbeam Queue supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-skiplist/Cargo.toml b/crossbeam-skiplist/Cargo.toml index 76edc460e..8708529c8 100644 --- a/crossbeam-skiplist/Cargo.toml +++ b/crossbeam-skiplist/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-skiplist" # - Create "crossbeam-skiplist-X.Y.Z" git tag version = "0.1.1" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-skiplist" diff --git a/crossbeam-skiplist/README.md b/crossbeam-skiplist/README.md index 787ce87a5..0a08e633d 100644 --- a/crossbeam-skiplist/README.md +++ b/crossbeam-skiplist/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-skiplist#license https://crates.io/crates/crossbeam-skiplist) [![Documentation](https://docs.rs/crossbeam-skiplist/badge.svg)]( https://docs.rs/crossbeam-skiplist) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -34,7 +34,7 @@ crossbeam-skiplist = "0.1" Crossbeam Skiplist supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-utils/Cargo.toml b/crossbeam-utils/Cargo.toml index a110da24e..dd7556b52 100644 --- a/crossbeam-utils/Cargo.toml +++ b/crossbeam-utils/Cargo.toml @@ -6,7 +6,7 @@ name = "crossbeam-utils" # - Create "crossbeam-utils-X.Y.Z" git tag version = "0.8.16" edition = "2018" -rust-version = "1.38" +rust-version = "1.59" license = "MIT OR Apache-2.0" repository = "https://github.com/crossbeam-rs/crossbeam" homepage = "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils" @@ -22,6 +22,7 @@ default = ["std"] std = [] [dependencies] +atomic-maybe-uninit = "0.2.22" cfg-if = "1" # Enable the use of loom for concurrency testing. diff --git a/crossbeam-utils/README.md b/crossbeam-utils/README.md index 5491c7213..b2c67de6a 100644 --- a/crossbeam-utils/README.md +++ b/crossbeam-utils/README.md @@ -8,7 +8,7 @@ https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils#license) https://crates.io/crates/crossbeam-utils) [![Documentation](https://docs.rs/crossbeam-utils/badge.svg)]( https://docs.rs/crossbeam-utils) -[![Rust 1.38+](https://img.shields.io/badge/rust-1.38+-lightgray.svg)]( +[![Rust 1.59+](https://img.shields.io/badge/rust-1.59+-lightgray.svg)]( https://www.rust-lang.org) [![chat](https://img.shields.io/discord/569610676205781012.svg?logo=discord)](https://discord.com/invite/JXYwgWZ) @@ -55,7 +55,7 @@ crossbeam-utils = "0.8" Crossbeam Utils supports stable Rust releases going back at least six months, and every time the minimum supported Rust version is increased, a new minor -version is released. Currently, the minimum supported Rust version is 1.38. +version is released. Currently, the minimum supported Rust version is 1.59. ## License diff --git a/crossbeam-utils/build.rs b/crossbeam-utils/build.rs index 39785f030..616f0ae62 100644 --- a/crossbeam-utils/build.rs +++ b/crossbeam-utils/build.rs @@ -64,4 +64,7 @@ fn main() { if sanitize.contains("thread") { println!("cargo:rustc-cfg=crossbeam_sanitize_thread"); } + if !sanitize.is_empty() { + println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback"); + } } diff --git a/crossbeam-utils/src/atomic/atomic_cell.rs b/crossbeam-utils/src/atomic/atomic_cell.rs index c60efbc94..658482cb4 100644 --- a/crossbeam-utils/src/atomic/atomic_cell.rs +++ b/crossbeam-utils/src/atomic/atomic_cell.rs @@ -1,7 +1,7 @@ // Necessary for implementing atomic methods for `AtomicUnit` #![allow(clippy::unit_arg)] -use crate::primitive::sync::atomic::{self, AtomicBool}; +use crate::primitive::sync::atomic; use core::cell::UnsafeCell; use core::cmp; use core::fmt; @@ -103,6 +103,8 @@ impl AtomicCell { /// # Examples /// /// ``` + /// # // Always use fallback for now on environments that do not support inline assembly. + /// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; } /// use crossbeam_utils::atomic::AtomicCell; /// /// // This type is internally represented as `AtomicUsize` so we can just use atomic @@ -294,8 +296,54 @@ impl Drop for AtomicCell { } } +macro_rules! atomic { + // If values of type `$t` can be transmuted into values of the primitive atomic type `$atomic`, + // declares variable `$a` of type `$atomic` and executes `$atomic_op`, breaking out of the loop. + (@check, $t:ty, $atomic:ty, $a:ident, $atomic_op:expr) => { + if can_transmute::<$t, $atomic>() { + let $a: &$atomic; + break $atomic_op; + } + }; + + // If values of type `$t` can be transmuted into values of a primitive atomic type, declares + // variable `$a` of that type and executes `$atomic_op`. Otherwise, just executes + // `$fallback_op`. + ($t:ty, $a:ident, $atomic_op:expr, $fallback_op:expr) => { + loop { + atomic!(@check, $t, AtomicUnit, $a, $atomic_op); + + // Always use fallback for now on environments that do not support inline assembly. + #[cfg(not(any( + miri, + crossbeam_loom, + crossbeam_atomic_cell_force_fallback, + )))] + { + atomic_maybe_uninit::cfg_has_atomic_8! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_16! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_32! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_64! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + atomic_maybe_uninit::cfg_has_atomic_128! { + atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit, $a, $atomic_op); + } + } + + break $fallback_op; + } + }; +} + macro_rules! impl_arithmetic { - ($t:ty, fallback, $example:tt) => { + ($t:ty, fetch_update, $example:tt) => { impl AtomicCell<$t> { /// Increments the current value by `val` and returns the previous value. /// @@ -313,11 +361,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_add(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = value.wrapping_add(val); - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(old.wrapping_add(val))).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_add(val); + old + } + } } /// Decrements the current value by `val` and returns the previous value. @@ -336,11 +392,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_sub(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = value.wrapping_sub(val); - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(old.wrapping_sub(val))).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_sub(val); + old + } + } } /// Applies bitwise "and" to the current value and returns the previous value. @@ -357,11 +421,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_and(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value &= val; - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(old & val)).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } + } } /// Applies bitwise "nand" to the current value and returns the previous value. @@ -378,11 +450,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_nand(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = !(old & val); - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(!(old & val))).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } + } } /// Applies bitwise "or" to the current value and returns the previous value. @@ -399,11 +479,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_or(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value |= val; - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(old | val)).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } + } } /// Applies bitwise "xor" to the current value and returns the previous value. @@ -420,11 +508,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_xor(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value ^= val; - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(old ^ val)).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } + } } /// Compares and sets the maximum of the current value and `val`, @@ -442,11 +538,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_max(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = cmp::max(old, val); - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(cmp::max(old, val))).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::max(old, val); + old + } + } } /// Compares and sets the minimum of the current value and `val`, @@ -464,15 +568,23 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_min(&self, val: $t) -> $t { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = cmp::min(old, val); - old + atomic! { + $t, _a, + { + self.fetch_update(|old| Some(cmp::min(old, val))).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::min(old, val); + old + } + } } } }; - ($t:ty, $atomic:ty, $example:tt) => { + ($t:ty, $atomic:ident, $example:tt) => { impl AtomicCell<$t> { /// Increments the current value by `val` and returns the previous value. /// @@ -490,15 +602,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_add(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - let a = unsafe { &*(self.as_ptr() as *const $atomic) }; - a.fetch_add(val, Ordering::AcqRel) - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = value.wrapping_add(val); - old + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_add(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_add(val); + old + } } } @@ -518,15 +634,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_sub(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - let a = unsafe { &*(self.as_ptr() as *const $atomic) }; - a.fetch_sub(val, Ordering::AcqRel) - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = value.wrapping_sub(val); - old + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_sub(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = value.wrapping_sub(val); + old + } } } @@ -544,15 +664,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_and(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - let a = unsafe { &*(self.as_ptr() as *const $atomic) }; - a.fetch_and(val, Ordering::AcqRel) - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value &= val; - old + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_and(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } } } @@ -570,15 +694,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_nand(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - let a = unsafe { &*(self.as_ptr() as *const $atomic) }; - a.fetch_nand(val, Ordering::AcqRel) - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = !(old & val); - old + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_nand(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } } } @@ -596,15 +724,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_or(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - let a = unsafe { &*(self.as_ptr() as *const $atomic) }; - a.fetch_or(val, Ordering::AcqRel) - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value |= val; - old + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_or(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } } } @@ -622,15 +754,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_xor(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - let a = unsafe { &*(self.as_ptr() as *const $atomic) }; - a.fetch_xor(val, Ordering::AcqRel) - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value ^= val; - old + atomic! { + $t, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; + a.fetch_xor(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } } } @@ -649,15 +785,19 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_max(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - // TODO: Atomic*::fetch_max requires Rust 1.45. - self.fetch_update(|old| Some(cmp::max(old, val))).unwrap() - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = cmp::max(old, val); - old + atomic! { + $t, _a, + { + // TODO: Atomic*::fetch_max requires Rust 1.45. + self.fetch_update(|old| Some(cmp::max(old, val))).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::max(old, val); + old + } } } @@ -676,51 +816,51 @@ macro_rules! impl_arithmetic { /// ``` #[inline] pub fn fetch_min(&self, val: $t) -> $t { - if can_transmute::<$t, $atomic>() { - // TODO: Atomic*::fetch_min requires Rust 1.45. - self.fetch_update(|old| Some(cmp::min(old, val))).unwrap() - } else { - let _guard = lock(self.as_ptr() as usize).write(); - let value = unsafe { &mut *(self.as_ptr()) }; - let old = *value; - *value = cmp::min(old, val); - old + atomic! { + $t, _a, + { + // TODO: Atomic*::fetch_min requires Rust 1.45. + self.fetch_update(|old| Some(cmp::min(old, val))).unwrap() + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = cmp::min(old, val); + old + } } } } }; } -impl_arithmetic!(u8, atomic::AtomicU8, "let a = AtomicCell::new(7u8);"); -impl_arithmetic!(i8, atomic::AtomicI8, "let a = AtomicCell::new(7i8);"); -impl_arithmetic!(u16, atomic::AtomicU16, "let a = AtomicCell::new(7u16);"); -impl_arithmetic!(i16, atomic::AtomicI16, "let a = AtomicCell::new(7i16);"); -impl_arithmetic!(u32, atomic::AtomicU32, "let a = AtomicCell::new(7u32);"); -impl_arithmetic!(i32, atomic::AtomicI32, "let a = AtomicCell::new(7i32);"); +impl_arithmetic!(u8, AtomicU8, "let a = AtomicCell::new(7u8);"); +impl_arithmetic!(i8, AtomicI8, "let a = AtomicCell::new(7i8);"); + +impl_arithmetic!(u16, AtomicU16, "let a = AtomicCell::new(7u16);"); +impl_arithmetic!(i16, AtomicI16, "let a = AtomicCell::new(7i16);"); + +impl_arithmetic!(u32, AtomicU32, "let a = AtomicCell::new(7u32);"); +impl_arithmetic!(i32, AtomicI32, "let a = AtomicCell::new(7i32);"); + #[cfg(not(crossbeam_no_atomic_64))] -impl_arithmetic!(u64, atomic::AtomicU64, "let a = AtomicCell::new(7u64);"); +impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);"); #[cfg(not(crossbeam_no_atomic_64))] -impl_arithmetic!(i64, atomic::AtomicI64, "let a = AtomicCell::new(7i64);"); +impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);"); #[cfg(crossbeam_no_atomic_64)] -impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);"); +impl_arithmetic!(u64, fetch_update, "let a = AtomicCell::new(7u64);"); #[cfg(crossbeam_no_atomic_64)] -impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);"); +impl_arithmetic!(i64, fetch_update, "let a = AtomicCell::new(7i64);"); + // TODO: AtomicU128 is unstable -// impl_arithmetic!(u128, atomic::AtomicU128, "let a = AtomicCell::new(7u128);"); -// impl_arithmetic!(i128, atomic::AtomicI128, "let a = AtomicCell::new(7i128);"); -impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);"); -impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);"); - -impl_arithmetic!( - usize, - atomic::AtomicUsize, - "let a = AtomicCell::new(7usize);" -); -impl_arithmetic!( - isize, - atomic::AtomicIsize, - "let a = AtomicCell::new(7isize);" -); +// impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);"); +// impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);"); +impl_arithmetic!(u128, fetch_update, "let a = AtomicCell::new(7u128);"); +impl_arithmetic!(i128, fetch_update, "let a = AtomicCell::new(7i128);"); + +impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);"); +impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);"); impl AtomicCell { /// Applies logical "and" to the current value and returns the previous value. @@ -740,8 +880,20 @@ impl AtomicCell { /// ``` #[inline] pub fn fetch_and(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_and(val, Ordering::AcqRel) + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_and(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value &= val; + old + } + } } /// Applies logical "nand" to the current value and returns the previous value. @@ -764,8 +916,20 @@ impl AtomicCell { /// ``` #[inline] pub fn fetch_nand(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_nand(val, Ordering::AcqRel) + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_nand(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value = !(old & val); + old + } + } } /// Applies logical "or" to the current value and returns the previous value. @@ -785,8 +949,20 @@ impl AtomicCell { /// ``` #[inline] pub fn fetch_or(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_or(val, Ordering::AcqRel) + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_or(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value |= val; + old + } + } } /// Applies logical "xor" to the current value and returns the previous value. @@ -806,8 +982,20 @@ impl AtomicCell { /// ``` #[inline] pub fn fetch_xor(&self, val: bool) -> bool { - let a = unsafe { &*(self.as_ptr() as *const AtomicBool) }; - a.fetch_xor(val, Ordering::AcqRel) + atomic! { + bool, _a, + { + let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; + a.fetch_xor(val, Ordering::AcqRel) + }, + { + let _guard = lock(self.as_ptr() as usize).write(); + let value = unsafe { &mut *(self.as_ptr()) }; + let old = *value; + *value ^= val; + old + } + } } } @@ -908,48 +1096,9 @@ impl AtomicUnit { } } -macro_rules! atomic { - // If values of type `$t` can be transmuted into values of the primitive atomic type `$atomic`, - // declares variable `$a` of type `$atomic` and executes `$atomic_op`, breaking out of the loop. - (@check, $t:ty, $atomic:ty, $a:ident, $atomic_op:expr) => { - if can_transmute::<$t, $atomic>() { - let $a: &$atomic; - break $atomic_op; - } - }; - - // If values of type `$t` can be transmuted into values of a primitive atomic type, declares - // variable `$a` of that type and executes `$atomic_op`. Otherwise, just executes - // `$fallback_op`. - ($t:ty, $a:ident, $atomic_op:expr, $fallback_op:expr) => { - loop { - atomic!(@check, $t, AtomicUnit, $a, $atomic_op); - - atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op); - atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op); - atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op); - #[cfg(not(crossbeam_no_atomic_64))] - atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op); - // TODO: AtomicU128 is unstable - // atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op); - - break $fallback_op; - } - }; -} - /// Returns `true` if operations on `AtomicCell` are lock-free. const fn atomic_is_lock_free() -> bool { - // HACK(taiki-e): This is equivalent to `atomic! { T, _a, true, false }`, but can be used in const fn even in our MSRV (Rust 1.38). - let is_lock_free = can_transmute::() - | can_transmute::() - | can_transmute::() - | can_transmute::(); - #[cfg(not(crossbeam_no_atomic_64))] - let is_lock_free = is_lock_free | can_transmute::(); - // TODO: AtomicU128 is unstable - // let is_lock_free = is_lock_free | can_transmute::(); - is_lock_free + atomic! { T, _a, true, false } } /// Atomically reads data from `src`. diff --git a/crossbeam-utils/src/atomic/mod.rs b/crossbeam-utils/src/atomic/mod.rs index 38967859f..1dd08edb3 100644 --- a/crossbeam-utils/src/atomic/mod.rs +++ b/crossbeam-utils/src/atomic/mod.rs @@ -4,7 +4,11 @@ //! * [`AtomicConsume`], for reading from primitive atomic types with "consume" ordering. #[cfg(not(crossbeam_no_atomic_cas))] +// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic +// types have a different in-memory representation than the underlying type. +// TODO: The latest loom supports fences, so fallback using seqlock may be available. #[cfg(not(crossbeam_loom))] +atomic_maybe_uninit::cfg_has_atomic_cas! { cfg_if::cfg_if! { // Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap // around. @@ -23,15 +27,9 @@ cfg_if::cfg_if! { } } -#[cfg(not(crossbeam_no_atomic_cas))] -// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic -// types have a different in-memory representation than the underlying type. -// TODO: The latest loom supports fences, so fallback using seqlock may be available. -#[cfg(not(crossbeam_loom))] mod atomic_cell; -mod consume; - -#[cfg(not(crossbeam_no_atomic_cas))] -#[cfg(not(crossbeam_loom))] pub use self::atomic_cell::AtomicCell; +} + +mod consume; pub use self::consume::AtomicConsume; diff --git a/crossbeam-utils/tests/atomic_cell.rs b/crossbeam-utils/tests/atomic_cell.rs index edb7a4bc0..847f87083 100644 --- a/crossbeam-utils/tests/atomic_cell.rs +++ b/crossbeam-utils/tests/atomic_cell.rs @@ -6,6 +6,15 @@ use crossbeam_utils::atomic::AtomicCell; #[test] fn is_lock_free() { + // Always use fallback for now on environments that do not support inline assembly. + if cfg!(any( + miri, + crossbeam_loom, + crossbeam_atomic_cell_force_fallback, + )) { + return; + } + struct UsizeWrap(usize); struct U8Wrap(bool); struct I16Wrap(i16); @@ -44,8 +53,9 @@ fn is_lock_free() { cfg!(not(crossbeam_no_atomic_64)) ); - // AtomicU128 is unstable - assert!(!AtomicCell::::is_lock_free()); + // TODO + // // AtomicU128 is unstable + // assert!(!AtomicCell::::is_lock_free()); } #[test] @@ -319,10 +329,11 @@ fn issue_748() { } assert_eq!(mem::size_of::(), 8); - assert_eq!( - AtomicCell::::is_lock_free(), - cfg!(not(crossbeam_no_atomic_64)) - ); + // TODO + // assert_eq!( + // AtomicCell::::is_lock_free(), + // cfg!(not(crossbeam_no_atomic_64)) + // ); let x = AtomicCell::new(Test::FieldLess); assert_eq!(x.load(), Test::FieldLess); }