From 15bca3b6829cf187dc6c1aa13f96488ded2561dd Mon Sep 17 00:00:00 2001 From: Rafal Bielski Date: Thu, 15 Aug 2024 16:40:09 +0100 Subject: [PATCH] [CUDA] Fix race condition in CUDA stream creation Do not increment NumComputeStreams / NumTransferStreams before cuStreamCreateWithPriority returns. Too early increment caused other threads to read the incremented count before a CUDA stream was created and try to use an invalid stream handle, causing crashes. The construction: ``` if (condition) { lock_this_scope if (condition) { create_object update_condition } } use_object ``` is only thread-safe if update_condition happens after create_object is completed. --- source/adapters/cuda/queue.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/adapters/cuda/queue.cpp b/source/adapters/cuda/queue.cpp index bd92a01400..e5cce87798 100644 --- a/source/adapters/cuda/queue.cpp +++ b/source/adapters/cuda/queue.cpp @@ -45,7 +45,8 @@ CUstream ur_queue_handle_t_::getNextComputeStream(uint32_t *StreamToken) { // change NumComputeStreams after that if (NumComputeStreams < ComputeStreams.size()) { UR_CHECK_ERROR(cuStreamCreateWithPriority( - &ComputeStreams[NumComputeStreams++], Flags, Priority)); + &ComputeStreams[NumComputeStreams], Flags, Priority)); + ++NumComputeStreams; } } Token = ComputeStreamIndex++; @@ -110,7 +111,8 @@ CUstream ur_queue_handle_t_::getNextTransferStream() { // change NumTransferStreams after that if (NumTransferStreams < TransferStreams.size()) { UR_CHECK_ERROR(cuStreamCreateWithPriority( - &TransferStreams[NumTransferStreams++], Flags, Priority)); + &TransferStreams[NumTransferStreams], Flags, Priority)); + ++NumTransferStreams; } } uint32_t StreamI = TransferStreamIndex++ % TransferStreams.size();