Abstract the state type for futexes

In the same way that we expose SmallAtomic and SmallPrimitive to allow
Windows to use a value other than an AtomicU32 for its futex state, this
patch switches the primary futex state type from AtomicU32 to
futex::Atomic.  The futex::Atomic type should be usable as an atomic
value with underlying primitive type equal to futex::Primitive.

This allows supporting the futex API on systems where the underlying
kernel futex implementation requires more state than simply an
AtomicU32.

All in-tree futex implementations simply define {Atomic,Primitive}
directly as {AtomicU32,u32}.

Author: paulmenage

library/std/src/sys/pal/hermit/futex.rs

@@ -3,6 +3,11 @@ use crate::ptr::null;
 use crate::sync::atomic::AtomicU32;
 use crate::time::Duration;
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pub type Atomic = AtomicU32;
+/// Must be the underlying type of Atomic
+pub type Primitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
 pub type SmallAtomic = AtomicU32;
 /// Must be the underlying type of SmallAtomic

library/std/src/sys/pal/unix/futex.rs

@@ -11,6 +11,11 @@
 use crate::sync::atomic::AtomicU32;
 use crate::time::Duration;
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pub type Atomic = AtomicU32;
+/// Must be the underlying type of Atomic
+pub type Primitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
 pub type SmallAtomic = AtomicU32;
 /// Must be the underlying type of SmallAtomic

library/std/src/sys/pal/wasm/atomics/futex.rs

@@ -6,6 +6,11 @@ use core::arch::wasm64 as wasm;
 use crate::sync::atomic::AtomicU32;
 use crate::time::Duration;
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pub type Atomic = AtomicU32;
+/// Must be the underlying type of Atomic
+pub type Primitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
 pub type SmallAtomic = AtomicU32;
 /// Must be the underlying type of SmallAtomic

library/std/src/sys/pal/windows/futex.rs

@@ -9,6 +9,11 @@ use core::{mem, ptr};
 use super::api::{self, WinError};
 use crate::sys::{c, dur2timeout};
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pub type Atomic = AtomicU32;
+/// Must be the underlying type of Atomic
+pub type Primitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
 pub type SmallAtomic = AtomicU8;
 /// Must be the underlying type of SmallAtomic

library/std/src/sys/sync/condvar/futex.rs

@@ -1,20 +1,19 @@
-use crate::sync::atomic::AtomicU32;
 use crate::sync::atomic::Ordering::Relaxed;
-use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
+use crate::sys::futex::{Atomic, futex_wait, futex_wake, futex_wake_all};
 use crate::sys::sync::Mutex;
 use crate::time::Duration;
 
 pub struct Condvar {
     // The value of this atomic is simply incremented on every notification.
     // This is used by `.wait()` to not miss any notifications after
     // unlocking the mutex and before waiting for notifications.
-    futex: AtomicU32,
+    futex: Atomic,
 }
 
 impl Condvar {
     #[inline]
     pub const fn new() -> Self {
-        Self { futex: AtomicU32::new(0) }
+        Self { futex: Atomic::new(0) }
     }
 
     // All the memory orderings here are `Relaxed`,

library/std/src/sys/sync/once/futex.rs

@@ -1,39 +1,38 @@
 use crate::cell::Cell;
 use crate::sync as public;
-use crate::sync::atomic::AtomicU32;
 use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
 use crate::sync::once::ExclusiveState;
-use crate::sys::futex::{futex_wait, futex_wake_all};
+use crate::sys::futex::{Atomic, Primitive, futex_wait, futex_wake_all};
 
 // On some platforms, the OS is very nice and handles the waiter queue for us.
 // This means we only need one atomic value with 4 states:
 
 /// No initialization has run yet, and no thread is currently using the Once.
-const INCOMPLETE: u32 = 0;
+const INCOMPLETE: Primitive = 0;
 /// Some thread has previously attempted to initialize the Once, but it panicked,
 /// so the Once is now poisoned. There are no other threads currently accessing
 /// this Once.
-const POISONED: u32 = 1;
+const POISONED: Primitive = 1;
 /// Some thread is currently attempting to run initialization. It may succeed,
 /// so all future threads need to wait for it to finish.
-const RUNNING: u32 = 2;
+const RUNNING: Primitive = 2;
 /// Initialization has completed and all future calls should finish immediately.
-const COMPLETE: u32 = 3;
+const COMPLETE: Primitive = 3;
 
 // An additional bit indicates whether there are waiting threads:
 
 /// May only be set if the state is not COMPLETE.
-const QUEUED: u32 = 4;
+const QUEUED: Primitive = 4;
 
 // Threads wait by setting the QUEUED bit and calling `futex_wait` on the state
 // variable. When the running thread finishes, it will wake all waiting threads using
 // `futex_wake_all`.
 
-const STATE_MASK: u32 = 0b11;
+const STATE_MASK: Primitive = 0b11;
 
 pub struct OnceState {
     poisoned: bool,
-    set_state_to: Cell<u32>,
+    set_state_to: Cell<Primitive>,
 }
 
 impl OnceState {
@@ -49,8 +48,8 @@ impl OnceState {
 }
 
 struct CompletionGuard<'a> {
-    state_and_queued: &'a AtomicU32,
-    set_state_on_drop_to: u32,
+    state_and_queued: &'a Atomic,
+    set_state_on_drop_to: Primitive,
 }
 
 impl<'a> Drop for CompletionGuard<'a> {
@@ -65,13 +64,13 @@ impl<'a> Drop for CompletionGuard<'a> {
 }
 
 pub struct Once {
-    state_and_queued: AtomicU32,
+    state_and_queued: Atomic,
 }
 
 impl Once {
     #[inline]
     pub const fn new() -> Once {
-        Once { state_and_queued: AtomicU32::new(INCOMPLETE) }
+        Once { state_and_queued: Atomic::new(INCOMPLETE) }
     }
 
     #[inline]

library/std/src/sys/sync/rwlock/futex.rs

@@ -1,6 +1,5 @@
-use crate::sync::atomic::AtomicU32;
 use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
-use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
+use crate::sys::futex::{Atomic, Primitive, futex_wait, futex_wake, futex_wake_all};
 
 pub struct RwLock {
     // The state consists of a 30-bit reader counter, a 'readers waiting' flag, and a 'writers waiting' flag.
@@ -10,41 +9,41 @@ pub struct RwLock {
     //   0x3FFF_FFFF: Write locked
     // Bit 30: Readers are waiting on this futex.
     // Bit 31: Writers are waiting on the writer_notify futex.
-    state: AtomicU32,
+    state: Atomic,
     // The 'condition variable' to notify writers through.
     // Incremented on every signal.
-    writer_notify: AtomicU32,
+    writer_notify: Atomic,
 }
 
-const READ_LOCKED: u32 = 1;
-const MASK: u32 = (1 << 30) - 1;
-const WRITE_LOCKED: u32 = MASK;
-const MAX_READERS: u32 = MASK - 1;
-const READERS_WAITING: u32 = 1 << 30;
-const WRITERS_WAITING: u32 = 1 << 31;
+const READ_LOCKED: Primitive = 1;
+const MASK: Primitive = (1 << 30) - 1;
+const WRITE_LOCKED: Primitive = MASK;
+const MAX_READERS: Primitive = MASK - 1;
+const READERS_WAITING: Primitive = 1 << 30;
+const WRITERS_WAITING: Primitive = 1 << 31;
 
 #[inline]
-fn is_unlocked(state: u32) -> bool {
+fn is_unlocked(state: Primitive) -> bool {
     state & MASK == 0
 }
 
 #[inline]
-fn is_write_locked(state: u32) -> bool {
+fn is_write_locked(state: Primitive) -> bool {
     state & MASK == WRITE_LOCKED
 }
 
 #[inline]
-fn has_readers_waiting(state: u32) -> bool {
+fn has_readers_waiting(state: Primitive) -> bool {
     state & READERS_WAITING != 0
 }
 
 #[inline]
-fn has_writers_waiting(state: u32) -> bool {
+fn has_writers_waiting(state: Primitive) -> bool {
     state & WRITERS_WAITING != 0
 }
 
 #[inline]
-fn is_read_lockable(state: u32) -> bool {
+fn is_read_lockable(state: Primitive) -> bool {
     // This also returns false if the counter could overflow if we tried to read lock it.
     //
     // We don't allow read-locking if there's readers waiting, even if the lock is unlocked
@@ -55,14 +54,14 @@ fn is_read_lockable(state: u32) -> bool {
 }
 
 #[inline]
-fn has_reached_max_readers(state: u32) -> bool {
+fn has_reached_max_readers(state: Primitive) -> bool {
     state & MASK == MAX_READERS
 }
 
 impl RwLock {
     #[inline]
     pub const fn new() -> Self {
-        Self { state: AtomicU32::new(0), writer_notify: AtomicU32::new(0) }
+        Self { state: Atomic::new(0), writer_notify: Atomic::new(0) }
     }
 
     #[inline]
@@ -225,7 +224,7 @@ impl RwLock {
     /// If both are waiting, this will wake up only one writer, but will fall
     /// back to waking up readers if there was no writer to wake up.
     #[cold]
-    fn wake_writer_or_readers(&self, mut state: u32) {
+    fn wake_writer_or_readers(&self, mut state: Primitive) {
         assert!(is_unlocked(state));
 
         // The readers waiting bit might be turned on at any point now,
@@ -290,7 +289,7 @@ impl RwLock {
 
     /// Spin for a while, but stop directly at the given condition.
     #[inline]
-    fn spin_until(&self, f: impl Fn(u32) -> bool) -> u32 {
+    fn spin_until(&self, f: impl Fn(Primitive) -> bool) -> Primitive {
         let mut spin = 100; // Chosen by fair dice roll.
         loop {
             let state = self.state.load(Relaxed);
@@ -303,13 +302,13 @@ impl RwLock {
     }
 
     #[inline]
-    fn spin_write(&self) -> u32 {
+    fn spin_write(&self) -> Primitive {
         // Stop spinning when it's unlocked or when there's waiting writers, to keep things somewhat fair.
         self.spin_until(|state| is_unlocked(state) || has_writers_waiting(state))
     }
 
     #[inline]
-    fn spin_read(&self) -> u32 {
+    fn spin_read(&self) -> Primitive {
         // Stop spinning when it's unlocked or read locked, or when there's waiting threads.
         self.spin_until(|state| {
             !is_write_locked(state) || has_readers_waiting(state) || has_writers_waiting(state)