From a55a10ef97461876c45863065891dfa81e10d2a8 Mon Sep 17 00:00:00 2001 From: "Jason P. Rahman" Date: Fri, 15 Mar 2024 05:07:40 +0000 Subject: [PATCH] Correctly handle coop_taskrun When coop taskrun or defer taskrun are set to defer running completion processing until the next system call (coop) or the start of the next io_uring_enter() call, the taskrun flag needs to be checked. The kernel will set the taskrun flag if there are deferred completions that need additional processing in kernel space before they're available to the application. Here we update the check in the submission path to ensure that flag is checked and GETEVENTS is passed into io_uring_enter() when needed --- src/submit.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/submit.rs b/src/submit.rs index 7db9b64..786c41b 100644 --- a/src/submit.rs +++ b/src/submit.rs @@ -63,10 +63,10 @@ impl<'a> Submitter<'a> { } } - /// CQ ring is overflown - fn sq_cq_overflow(&self) -> bool { + /// CQ ring is overflown or a deferred task is runnable. Either case requires entering the kernel + fn sq_cq_overflow_or_taskrun(&self) -> bool { unsafe { - (*self.sq_flags).load(atomic::Ordering::Acquire) & sys::IORING_SQ_CQ_OVERFLOW != 0 + (*self.sq_flags).load(atomic::Ordering::Acquire) & (sys::IORING_SQ_CQ_OVERFLOW | sys::IORING_SQ_TASKRUN) != 0 } } @@ -115,12 +115,12 @@ impl<'a> Submitter<'a> { let len = self.sq_len(); let mut flags = 0; - // This logic suffers from the fact the sq_cq_overflow and sq_need_wakeup + // This logic suffers from the fact the sq_cq_overflow_or_taskrun and sq_need_wakeup // each cause an atomic load of the same variable, self.sq_flags. // In the hottest paths, when a server is running with sqpoll, // this is going to be hit twice, when once would be sufficient. - if want > 0 || self.params.is_setup_iopoll() || self.sq_cq_overflow() { + if want > 0 || self.params.is_setup_iopoll() || self.sq_cq_overflow_or_taskrun() { flags |= sys::IORING_ENTER_GETEVENTS; } @@ -145,7 +145,7 @@ impl<'a> Submitter<'a> { let len = self.sq_len(); let mut flags = sys::IORING_ENTER_EXT_ARG; - if want > 0 || self.params.is_setup_iopoll() || self.sq_cq_overflow() { + if want > 0 || self.params.is_setup_iopoll() || self.sq_cq_overflow_or_taskrun() { flags |= sys::IORING_ENTER_GETEVENTS; }