diff --git a/.github/bors.toml b/.github/bors.toml
index 6546ca24a..731db26c4 100644
--- a/.github/bors.toml
+++ b/.github/bors.toml
@@ -4,4 +4,4 @@ status = [
"test (aarch64-apple-darwin)",
"valgrind (aarch64-unknown-linux-gnu)",
]
-timeout_sec = 7200
+timeout_sec = 10800
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e36ed5772..75cbcac02 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -210,7 +210,7 @@ jobs:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+lse ${{ env.RANDOMIZE_LAYOUT }}
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+lse ${{ env.RANDOMIZE_LAYOUT }}
if: startsWith(matrix.target, 'aarch64') && startsWith(matrix.rust, 'nightly') && !(contains(matrix.target, '-musl') || contains(matrix.target, '-android'))
- # TODO: it seems qemu-user has not yet properly implemented FEAT_LSE2: https://github.com/taiki-e/portable-atomic/pull/11#issuecomment-1114044327
+ # TODO: As of QEMU 7.2, QEMU has not yet implemented FEAT_LSE2: https://linaro.atlassian.net/browse/QEMU-300
# pwr7
# powerpc64- (big-endian) is skipped because it is pre-pwr8 by default
diff --git a/build.rs b/build.rs
index f8bee3913..c1c8ac5d8 100644
--- a/build.rs
+++ b/build.rs
@@ -242,6 +242,8 @@ fn main() {
}
}
// lqarx and stqcx.
+ // Note: As of rustc 1.67, target_feature "quadword-atomics" is not available on rustc side:
+ // https://github.com/rust-lang/rust/blob/1.67.0/compiler/rustc_codegen_ssa/src/target_features.rs#L215
target_feature_if("quadword-atomics", has_pwr8_features, &version, None, false);
}
_ => {}
diff --git a/src/imp/atomic128/README.md b/src/imp/atomic128/README.md
index 217ae7741..2c9a9ab46 100644
--- a/src/imp/atomic128/README.md
+++ b/src/imp/atomic128/README.md
@@ -2,11 +2,13 @@
The table of targets that support 128-bit atomics and the instructions used:
-| target_arch | load | store | CAS | note |
-| ----------- | ----- | ----- | ---- | ---- |
-| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires nightly |
-| aarch64 | ldxp/stxp or casp or ldp | ldxp/stxp or stp | ldxp/stxp or casp | casp requires lse target feature, ldp/stp requires lse2 target feature.
Both compile-time and run-time detection are supported for lse. lse2 is currently compile-time detection only.
Requires rustc 1.59+ |
-| powerpc64 | lq | stq | lqarx/stqcx. | Little endian or target CPU pwr8+.
Requires nightly |
-| s390x | lpq | stpq | cdsg | Requires nightly |
+| target_arch | load | store | CAS | RMW | note |
+| ----------- | ----- | ----- | ---- | ---- | ---- |
+| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires nightly |
+| aarch64 | ldxp/stxp or casp or ldp | ldxp/stxp or casp or stp | ldxp/stxp or casp | ldxp/stxp or casp | casp requires lse target feature, ldp/stp requires lse2 target feature.
Both compile-time and run-time detection are supported for lse. lse2 is currently compile-time detection only.
Requires rustc 1.59+ |
+| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Little endian or target CPU pwr8+.
Requires nightly |
+| s390x | lpq | stpq | cdsg | cdsg | Requires nightly |
Run-time detections are enabled by default and can be disabled with `--cfg portable_atomic_no_outline_atomics`.
+
+See [aarch64.rs](aarch64.rs) module-level comments for more details on the instructions used on aarch64.
diff --git a/src/imp/atomic128/aarch64.rs b/src/imp/atomic128/aarch64.rs
index c7e45b5d6..148b505ac 100644
--- a/src/imp/atomic128/aarch64.rs
+++ b/src/imp/atomic128/aarch64.rs
@@ -3,19 +3,18 @@
// There are a few ways to implement 128-bit atomic operations in AArch64.
//
// - LDXP/STXP loop (DW LL/SC)
-// - CASP (DWCAS) added as FEAT_LSE (armv8.1-a)
-// - LDP/STP (DW load/store) if FEAT_LSE2 (armv8.4-a) is available
+// - CASP (DWCAS) added as FEAT_LSE (mandatory from armv8.1-a)
+// - LDP/STP (DW load/store) if FEAT_LSE2 (optional from armv8.2-a, mandatory from armv8.4-a) is available
//
-// If outline-atomics is not enabled, we use CASP if FEAT_LSE is enabled at
-// compile-time, otherwise, use LDXP/STXP loop.
-// If outline-atomics is enabled, we use CASP for load/compare_exchange(_weak) if
-// FEAT_LSE is available at run-time.
+// If outline-atomics is not enabled and FEAT_LSE is not available at
+// compile-time, we use LDXP/STXP loop.
+// If outline-atomics is enabled and FEAT_LSE is not available at
+// compile-time, we use CASP for CAS if FEAT_LSE is available
+// at run-time, otherwise, use LDXP/STXP loop.
+// If FEAT_LSE is available at compile-time, we use CASP for load/store/CAS/RMW.
// If FEAT_LSE2 is available at compile-time, we use LDP/STP for load/store.
//
-// Note: As of rustc 1.68.0-nightly, -C target-feature=+lse2 does not implicitly
-// enable target_feature "lse": https://godbolt.org/z/GYTcTeda6
-// Also, as of rustc 1.67, target_feature "lse2" is not available on rustc side:
-// https://github.com/rust-lang/rust/blob/1.67.0/compiler/rustc_codegen_ssa/src/target_features.rs#L47
+// Note: FEAT_LSE2 doesn't imply FEAT_LSE.
//
// Note that we do not separate LL and SC into separate functions, but handle
// them within a single asm block. This is because it is theoretically possible
@@ -42,9 +41,9 @@
// - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit
//
// Generated asm:
-// - aarch64 https://godbolt.org/z/z5cd5W8fh
-// - aarch64 (+lse) https://godbolt.org/z/offxb8rrj
-// - aarch64 (+lse,+lse2) https://godbolt.org/z/8nn4WE4cj
+// - aarch64 https://godbolt.org/z/8E1PanvhY
+// - aarch64 (+lse) https://godbolt.org/z/4377cG4T7
+// - aarch64 (+lse,+lse2) https://godbolt.org/z/6hsdMfKWv
include!("macros.rs");
@@ -147,24 +146,24 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
unsafe {
_atomic_load_ldp(src, order)
}
- #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
#[cfg(not(any(target_feature = "lse2", portable_atomic_target_feature = "lse2")))]
- // SAFETY: the caller must uphold the safety contract for `atomic_load`.
- // cfg guarantee that the CPU supports FEAT_LSE.
- unsafe {
- _atomic_compare_exchange_casp(src, 0, 0, order)
- }
- #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
- #[cfg(not(any(target_feature = "lse2", portable_atomic_target_feature = "lse2")))]
- // SAFETY: the caller must uphold the safety contract for `atomic_load`.
- unsafe {
- _atomic_load_ldxp_stxp(src, order)
+ {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_load`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ _atomic_compare_exchange_casp(src, 0, 0, order)
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_load`.
+ unsafe {
+ _atomic_load_ldxp_stxp(src, order)
+ }
}
}
// If CPU supports FEAT_LSE2, LDP is single-copy atomic reads,
// otherwise it is two single-copy atomic reads.
// Refs: B2.2.1 of the Arm Architecture Reference Manual Armv8, for Armv8-A architecture profile
-#[cfg(any(target_feature = "lse2", portable_atomic_target_feature = "lse2", test))]
#[inline]
unsafe fn _atomic_load_ldp(src: *mut u128, order: Ordering) -> u128 {
debug_assert!(src as usize % 16 == 0);
@@ -199,6 +198,8 @@ unsafe fn _atomic_load_ldp(src: *mut u128, order: Ordering) -> u128 {
}
#[inline]
unsafe fn _atomic_load_ldxp_stxp(src: *mut u128, order: Ordering) -> u128 {
+ debug_assert!(src as usize % 16 == 0);
+
// SAFETY: the caller must uphold the safety contract for `atomic_swap`.
unsafe {
let (mut prev_lo, mut prev_hi);
@@ -454,11 +455,49 @@ unsafe fn _atomic_compare_exchange_ldxp_stxp(
// so we always use strong CAS for now.
use self::atomic_compare_exchange as atomic_compare_exchange_weak;
+#[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse",))]
+#[inline]
+unsafe fn atomic_update_casp(dst: *mut u128, order: Ordering, mut f: F) -> u128
+where
+ F: FnMut(u128) -> u128,
+{
+ let failure = crate::utils::strongest_failure_ordering(order);
+ // SAFETY: the caller must uphold the safety contract for `atomic_update`.
+ unsafe {
+ // If FEAT_LSE2 is not supported, this works like byte-wise atomic.
+ // This is not single-copy atomic reads, but this is ok because subsequent
+ // CAS will check for consistency.
+ let mut old = _atomic_load_ldp(dst, failure);
+ loop {
+ let next = f(old);
+ let x = _atomic_compare_exchange_casp(dst, old, next, order);
+ if x == old {
+ return x;
+ }
+ old = x;
+ }
+ }
+}
+
#[inline]
unsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_swap`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |_| val)
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_swap`.
+ unsafe {
+ _atomic_swap_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_swap_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_swap`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_swap_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -487,9 +526,23 @@ unsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_add`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |x| x.wrapping_add(val))
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_add`.
+ unsafe {
+ _atomic_add_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_add_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_add`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_add_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -523,9 +576,23 @@ unsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_sub`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |x| x.wrapping_sub(val))
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_sub`.
+ unsafe {
+ _atomic_sub_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_sub_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_sub`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_sub_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -559,9 +626,23 @@ unsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_and`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |x| x & val)
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_and`.
+ unsafe {
+ _atomic_and_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_and_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_and`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_and_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -594,9 +675,23 @@ unsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_nand`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |x| !(x & val))
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_nand`.
+ unsafe {
+ _atomic_nand_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_nand_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_nand`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_nand_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -631,9 +726,23 @@ unsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_or`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |x| x | val)
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_or`.
+ unsafe {
+ _atomic_or_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_or_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_or`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_or_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -666,9 +775,23 @@ unsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_xor`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |x| x ^ val)
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_xor`.
+ unsafe {
+ _atomic_xor_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_xor_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_xor`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_xor_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -701,9 +824,23 @@ unsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_not`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst, order, |x| !x)
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_not`.
+ unsafe {
+ _atomic_not_ldxp_stxp(dst, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_not_ldxp_stxp(dst: *mut u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_not`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_not_ldxp_stxp`.
unsafe {
let (mut prev_lo, mut prev_hi);
macro_rules! not {
@@ -733,9 +870,24 @@ unsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_max(dst: *mut i128, val: i128, order: Ordering) -> i128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
+ // SAFETY: the caller must uphold the safety contract for `atomic_max`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst.cast(), order, |x| core::cmp::max(x as i128, val) as u128) as i128
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_max`.
+ unsafe {
+ _atomic_max_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_max_ldxp_stxp(dst: *mut i128, val: i128, order: Ordering) -> i128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_max`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_max_ldxp_stxp`.
unsafe {
let val = I128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -808,9 +960,23 @@ unsafe fn atomic_max(dst: *mut i128, val: i128, order: Ordering) -> i128 {
#[inline]
unsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_umax`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst.cast(), order, |x| core::cmp::max(x, val))
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_umax`.
+ unsafe {
+ _atomic_umax_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_umax_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_umax`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_umax_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -883,9 +1049,24 @@ unsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {
#[inline]
unsafe fn atomic_min(dst: *mut i128, val: i128, order: Ordering) -> i128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
+ // SAFETY: the caller must uphold the safety contract for `atomic_min`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst.cast(), order, |x| core::cmp::min(x as i128, val) as u128) as i128
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_min`.
+ unsafe {
+ _atomic_min_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_min_ldxp_stxp(dst: *mut i128, val: i128, order: Ordering) -> i128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_min`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_min_ldxp_stxp`.
unsafe {
let val = I128 { whole: val };
let (mut prev_lo, mut prev_hi);
@@ -958,9 +1139,23 @@ unsafe fn atomic_min(dst: *mut i128, val: i128, order: Ordering) -> i128 {
#[inline]
unsafe fn atomic_umin(dst: *mut u128, val: u128, order: Ordering) -> u128 {
+ #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_umin`.
+ // cfg guarantee that the CPU supports FEAT_LSE.
+ unsafe {
+ atomic_update_casp(dst.cast(), order, |x| core::cmp::min(x, val))
+ }
+ #[cfg(not(any(target_feature = "lse", portable_atomic_target_feature = "lse")))]
+ // SAFETY: the caller must uphold the safety contract for `atomic_umin`.
+ unsafe {
+ _atomic_umin_ldxp_stxp(dst, val, order)
+ }
+}
+#[inline]
+unsafe fn _atomic_umin_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {
debug_assert!(dst as usize % 16 == 0);
- // SAFETY: the caller must uphold the safety contract for `atomic_umin`.
+ // SAFETY: the caller must uphold the safety contract for `_atomic_umin_ldxp_stxp`.
unsafe {
let val = U128 { whole: val };
let (mut prev_lo, mut prev_hi);
diff --git a/src/imp/atomic128/intrinsics.rs b/src/imp/atomic128/intrinsics.rs
index 02a838016..8358b0c24 100644
--- a/src/imp/atomic128/intrinsics.rs
+++ b/src/imp/atomic128/intrinsics.rs
@@ -308,6 +308,7 @@ where
unsafe fn atomic_max(dst: *mut i128, val: i128, order: Ordering) -> i128 {
// LLVM 15 doesn't support 128-bit atomic min/max for powerpc64.
#[cfg(target_arch = "powerpc64")]
+ #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: the caller must uphold the safety contract for `atomic_max`
unsafe {
atomic_update(dst.cast(), order, |x| core::cmp::max(x as i128, val) as u128) as i128
@@ -335,6 +336,7 @@ unsafe fn atomic_max(dst: *mut i128, val: i128, order: Ordering) -> i128 {
unsafe fn atomic_min(dst: *mut i128, val: i128, order: Ordering) -> i128 {
// LLVM 15 doesn't support 128-bit atomic min/max for powerpc64.
#[cfg(target_arch = "powerpc64")]
+ #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]
// SAFETY: the caller must uphold the safety contract for `atomic_min`
unsafe {
atomic_update(dst.cast(), order, |x| core::cmp::min(x as i128, val) as u128) as i128
diff --git a/tools/build.sh b/tools/build.sh
index 6cdd2fc5b..1ced1eb4b 100755
--- a/tools/build.sh
+++ b/tools/build.sh
@@ -344,6 +344,7 @@ build() {
CARGO_TARGET_DIR="${target_dir}/lse" \
RUSTFLAGS="${target_rustflags} -C target-feature=+lse" \
x_cargo "${args[@]}" "$@"
+ # FEAT_LSE2 doesn't imply FEAT_LSE.
CARGO_TARGET_DIR="${target_dir}/lse2" \
RUSTFLAGS="${target_rustflags} -C target-feature=+lse,+lse2" \
x_cargo "${args[@]}" "$@"