time: add alternative timer for better multicore scalability

.github/workflows/ci.yml

@@ -1058,6 +1058,7 @@ jobs:
           CARGO_TARGET_WASM32_WASIP1_RUNNER: "wasmtime run --"
           CARGO_TARGET_WASM32_WASIP1_THREADS_RUNNER: "wasmtime run -W bulk-memory=y -W threads=y -S threads=y --"
           RUSTFLAGS: --cfg tokio_unstable -Dwarnings -C target-feature=+atomics,+bulk-memory -C link-args=--max-memory=67108864
+          RUSTDOCFLAGS: -C link-args=--max-memory=67108864
 
       - name: WASI test tokio-stream
         run: cargo test -p tokio-stream --target ${{ matrix.target }} --features time,net,io-util,sync

.github/workflows/loom.yml

@@ -52,7 +52,7 @@ jobs:
             toolchain: ${{ env.rust_stable }}
       - uses: Swatinem/rust-cache@v2
       - name: run tests
-        run: cargo test --lib --release --features full -- --nocapture runtime::time::tests
+        run: cargo test --lib --release --features full -- --nocapture runtime::time
         working-directory: tokio
 
   loom-current-thread:

spellcheck.dic

@@ -1,4 +1,4 @@
-308
+311
 &
 +
 <
@@ -64,6 +64,7 @@ codec
 codecs
 combinator
 combinators
+condvar
 config
 Config
 connectionless
@@ -163,6 +164,7 @@ Lauck
 libc
 lifecycle
 lifo
+LLVM
 lookups
 macOS
 MacOS
@@ -307,3 +309,4 @@ Wakers
 wakeup
 wakeups
 workstealing
+ZST

tokio/Cargo.toml

@@ -140,6 +140,7 @@ tokio-test = { version = "0.4.0", path = "../tokio-test" }
 tokio-stream = { version = "0.1", path = "../tokio-stream" }
 tokio-util = { version = "0.7", path = "../tokio-util", features = ["rt"] }
 futures = { version = "0.3.0", features = ["async-await"] }
+futures-test = "0.3.31"
 mockall = "0.13.0"
 async-stream = "0.3"
 futures-concurrency = "7.6.3"

tokio/src/runtime/builder.rs

@@ -1,7 +1,9 @@
 #![cfg_attr(loom, allow(unused_imports))]
 
 use crate::runtime::handle::Handle;
-use crate::runtime::{blocking, driver, Callback, HistogramBuilder, Runtime, TaskCallback};
+use crate::runtime::{
+    blocking, driver, Callback, HistogramBuilder, Runtime, TaskCallback, TimerFlavor,
+};
 #[cfg(tokio_unstable)]
 use crate::runtime::{metrics::HistogramConfiguration, LocalOptions, LocalRuntime, TaskMeta};
 use crate::util::rand::{RngSeed, RngSeedGenerator};
@@ -133,6 +135,8 @@ pub struct Builder {
 
     #[cfg(tokio_unstable)]
     pub(super) unhandled_panic: UnhandledPanic,
+
+    timer_flavor: TimerFlavor,
 }
 
 cfg_unstable! {
@@ -318,6 +322,8 @@ impl Builder {
             metrics_poll_count_histogram: HistogramBuilder::default(),
 
             disable_lifo_slot: false,
+
+            timer_flavor: TimerFlavor::Traditional,
         }
     }
 
@@ -363,6 +369,41 @@ impl Builder {
         self
     }
 
+    /// Enables the alternative timer implementation, which is disabled by default.
+    ///
+    /// The alternative timer implementation is an unstable feature that may
+    /// provide better performance on multi-threaded runtimes with a large number
+    /// of worker threads.
+    ///
+    /// This option only applies to multi-threaded runtimes. Attempting to use
+    /// this option with any other runtime type will have no effect.
+    ///
+    /// [Click here to share your experience with the alternative timer](https://github.com/tokio-rs/tokio/issues/7745)
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[cfg(not(target_family = "wasm"))]
+    /// # {
+    /// use tokio::runtime;
+    ///
+    /// let rt = runtime::Builder::new_multi_thread()
+    ///   .enable_alt_timer()
+    ///   .build()
+    ///   .unwrap();
+    /// # }
+    /// ```
+    #[cfg(all(tokio_unstable, feature = "time", feature = "rt-multi-thread"))]
+    #[cfg_attr(
+        docsrs,
+        doc(cfg(all(tokio_unstable, feature = "time", feature = "rt-multi-thread")))
+    )]
+    pub fn enable_alt_timer(&mut self) -> &mut Self {
+        self.enable_time();
+        self.timer_flavor = TimerFlavor::Alternative;
+        self
+    }
+
     /// Sets the number of worker threads the `Runtime` will use.
     ///
     /// This can be any number above 0 though it is advised to keep this value
@@ -992,6 +1033,7 @@ impl Builder {
             enable_time: self.enable_time,
             start_paused: self.start_paused,
             nevents: self.nevents,
+            timer_flavor: self.timer_flavor,
         }
     }
 
@@ -1544,7 +1586,9 @@ impl Builder {
         use crate::runtime::scheduler;
         use crate::runtime::Config;
 
-        let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?;
+        let mut cfg = self.get_cfg();
+        cfg.timer_flavor = TimerFlavor::Traditional;
+        let (driver, driver_handle) = driver::Driver::new(cfg)?;
 
         // Blocking pool
         let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads);
@@ -1761,6 +1805,7 @@ cfg_rt_multi_thread! {
                     seed_generator: seed_generator_1,
                     metrics_poll_count_histogram: self.metrics_poll_count_histogram_builder(),
                 },
+                self.timer_flavor,
             );
 
             let handle = Handle { inner: scheduler::Handle::MultiThread(handle) };

tokio/src/runtime/driver.rs

@@ -40,6 +40,7 @@ pub(crate) struct Cfg {
     pub(crate) enable_pause_time: bool,
     pub(crate) start_paused: bool,
     pub(crate) nevents: usize,
+    pub(crate) timer_flavor: crate::runtime::TimerFlavor,
 }
 
 impl Driver {
@@ -48,7 +49,8 @@ impl Driver {
 
         let clock = create_clock(cfg.enable_pause_time, cfg.start_paused);
 
-        let (time_driver, time_handle) = create_time_driver(cfg.enable_time, io_stack, &clock);
+        let (time_driver, time_handle) =
+            create_time_driver(cfg.enable_time, cfg.timer_flavor, io_stack, &clock);
 
         Ok((
             Self { inner: time_driver },
@@ -113,6 +115,14 @@ impl Handle {
                 .expect("A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers.")
         }
 
+        #[cfg(tokio_unstable)]
+        pub(crate) fn with_time<F, R>(&self, f: F) -> R
+        where
+            F: FnOnce(Option<&crate::runtime::time::Handle>) -> R,
+        {
+            f(self.time.as_ref())
+        }
+
         pub(crate) fn clock(&self) -> &Clock {
             &self.clock
         }
@@ -281,6 +291,7 @@ cfg_time! {
         Enabled {
             driver: crate::runtime::time::Driver,
         },
+        EnabledAlt(IoStack),
         Disabled(IoStack),
     }
 
@@ -293,13 +304,21 @@ cfg_time! {
 
     fn create_time_driver(
         enable: bool,
+        timer_flavor: crate::runtime::TimerFlavor,
         io_stack: IoStack,
         clock: &Clock,
     ) -> (TimeDriver, TimeHandle) {
         if enable {
-            let (driver, handle) = crate::runtime::time::Driver::new(io_stack, clock);
-
-            (TimeDriver::Enabled { driver }, Some(handle))
+            match timer_flavor {
+                crate::runtime::TimerFlavor::Traditional => {
+                    let (driver, handle) = crate::runtime::time::Driver::new(io_stack, clock);
+                    (TimeDriver::Enabled { driver }, Some(handle))
+                }
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                crate::runtime::TimerFlavor::Alternative => {
+                    (TimeDriver::EnabledAlt(io_stack), Some(crate::runtime::time::Driver::new_alt(clock)))
+                }
+            }
         } else {
             (TimeDriver::Disabled(io_stack), None)
         }
@@ -309,20 +328,23 @@ cfg_time! {
         pub(crate) fn park(&mut self, handle: &Handle) {
             match self {
                 TimeDriver::Enabled { driver, .. } => driver.park(handle),
+                TimeDriver::EnabledAlt(v) => v.park(handle),
                 TimeDriver::Disabled(v) => v.park(handle),
             }
         }
 
         pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) {
             match self {
                 TimeDriver::Enabled { driver } => driver.park_timeout(handle, duration),
+                TimeDriver::EnabledAlt(v) => v.park_timeout(handle, duration),
                 TimeDriver::Disabled(v) => v.park_timeout(handle, duration),
             }
         }
 
         pub(crate) fn shutdown(&mut self, handle: &Handle) {
             match self {
                 TimeDriver::Enabled { driver } => driver.shutdown(handle),
+                TimeDriver::EnabledAlt(v) => v.shutdown(handle),
                 TimeDriver::Disabled(v) => v.shutdown(handle),
             }
         }
@@ -341,6 +363,7 @@ cfg_not_time! {
 
     fn create_time_driver(
         _enable: bool,
+        _timer_flavor: crate::runtime::TimerFlavor,
         io_stack: IoStack,
         _clock: &Clock,
     ) -> (TimeDriver, TimeHandle) {

tokio/src/runtime/mod.rs

@@ -389,8 +389,135 @@ cfg_process_driver! {
     mod process;
 }
 
+#[cfg_attr(not(feature = "time"), allow(dead_code))]
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub(crate) enum TimerFlavor {
+    Traditional,
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+    Alternative,
+}
+
 cfg_time! {
     pub(crate) mod time;
+
+    #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+    pub(crate) mod time_alt;
+
+    use std::task::{Context, Poll};
+    use std::pin::Pin;
+
+    #[derive(Debug)]
+    pub(crate) enum Timer {
+        Traditional(time::TimerEntry),
+
+        #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+        Alternative(time_alt::Timer),
+    }
+
+    impl Timer {
+        #[track_caller]
+        pub(crate) fn new(
+            handle: crate::runtime::scheduler::Handle,
+            deadline: crate::time::Instant,
+        ) -> Self {
+            match handle.timer_flavor() {
+                crate::runtime::TimerFlavor::Traditional => {
+                    Timer::Traditional(time::TimerEntry::new(handle, deadline))
+                }
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                crate::runtime::TimerFlavor::Alternative => {
+                    Timer::Alternative(time_alt::Timer::new(handle, deadline))
+                }
+            }
+        }
+
+        pub(crate) fn deadline(&self) -> crate::time::Instant {
+            match self {
+                Timer::Traditional(entry) => entry.deadline(),
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(entry) => entry.deadline(),
+            }
+        }
+
+        pub(crate) fn is_elapsed(&self) -> bool {
+            match self {
+                Timer::Traditional(entry) => entry.is_elapsed(),
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(entry) => entry.is_elapsed(),
+            }
+        }
+
+        pub(crate) fn flavor(self: Pin<&Self>) -> TimerFlavor {
+            match self.get_ref() {
+                Timer::Traditional(_) => TimerFlavor::Traditional,
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(_) => TimerFlavor::Alternative,
+            }
+        }
+
+        pub(crate) fn reset(
+            self: Pin<&mut Self>,
+            new_time: crate::time::Instant,
+            reregister: bool
+        ) {
+            // Safety: we never move the inner entries.
+            let this = unsafe { self.get_unchecked_mut() };
+            match this {
+                Timer::Traditional(entry) => {
+                    // Safety: we never move the inner entries.
+                    unsafe { Pin::new_unchecked(entry).reset(new_time, reregister); }
+                }
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(_) => panic!("not implemented yet"),
+            }
+        }
+
+        pub(crate) fn poll_elapsed(
+            self: Pin<&mut Self>,
+            cx: &mut Context<'_>,
+        ) -> Poll<Result<(), crate::time::error::Error>> {
+            // Safety: we never move the inner entries.
+            let this = unsafe { self.get_unchecked_mut() };
+            match this {
+                Timer::Traditional(entry) => {
+                    // Safety: we never move the inner entries.
+                    unsafe { Pin::new_unchecked(entry).poll_elapsed(cx) }
+                }
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(entry) => {
+                    // Safety: we never move the inner entries.
+                    unsafe { Pin::new_unchecked(entry).poll_elapsed(cx).map(Ok) }
+                }
+            }
+        }
+
+        #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+        pub(crate) fn scheduler_handle(&self) -> &crate::runtime::scheduler::Handle {
+            match self {
+                Timer::Traditional(_) => unreachable!("we should not call this on Traditional Timer"),
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(entry) => entry.scheduler_handle(),
+            }
+        }
+
+        #[cfg(all(tokio_unstable, feature = "tracing"))]
+        pub(crate) fn driver(self: Pin<&Self>) -> &crate::runtime::time::Handle {
+            match self.get_ref() {
+                Timer::Traditional(entry) => entry.driver(),
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(entry) => entry.driver(),
+            }
+        }
+
+        #[cfg(all(tokio_unstable, feature = "tracing"))]
+        pub(crate) fn clock(self: Pin<&Self>) -> &crate::time::Clock {
+            match self.get_ref() {
+                Timer::Traditional(entry) => entry.clock(),
+                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
+                Timer::Alternative(entry) => entry.clock(),
+            }
+        }
+    }
 }
 
 cfg_signal_internal_and_unix! {