[SYCL] Fix segfault on program exit when user thread is not finished yet

sycl/source/detail/global_handler.cpp

@@ -28,17 +28,24 @@ namespace sycl {
 __SYCL_INLINE_VER_NAMESPACE(_V1) {
 namespace detail {
 
+using LockGuard = std::lock_guard<SpinLock>;
+
+GlobalHandler *GlobalHandler::MSyclGlobalObjectsHandler = new GlobalHandler();
+SpinLock GlobalHandler::MSyclGlobalHandlerProtector{};
+
 // Utility class to track references on object.
-// Used for Scheduler now and created as thread_local object.
-// Origin idea is to track usage of Scheduler from main and other used threads -
-// they increment MCounter; and to use but not add extra reference by our
-// thread_pool threads. For this control MIncrementCounter class member is used.
+// Used for GlobalHandler now and created as thread_local object on the first
+// Scheduler usage. Origin idea is to track usage of Scheduler from main and
+// other used threads - they increment MCounter; and to use but not add extra
+// reference by our thread_pool threads. For this control MIncrementCounter
+// class member is used.
 template <class ResourceHandler> class ObjectUsageCounter {
 public:
   // Note: -Wctad-maybe-unsupported may generate warning if no ResourceHandler
   // type explicitly declared.
-  ObjectUsageCounter(std::unique_ptr<ResourceHandler> &Obj, bool ModifyCounter)
-      : MModifyCounter(ModifyCounter), MObj(Obj) {
+  ObjectUsageCounter(ResourceHandler *&Obj, SpinLock &ObjProtector,
+                     bool ModifyCounter)
+      : MModifyCounter(ModifyCounter), MObj(Obj), MObjProtector(ObjProtector) {
     if (MModifyCounter)
       MCounter++;
   }
@@ -47,26 +54,31 @@ template <class ResourceHandler> class ObjectUsageCounter {
       return;
 
     MCounter--;
-    if (!MCounter && MObj)
-      MObj->releaseResources();
+    if (!MCounter) {
+      LockGuard Guard(MObjProtector);
+      if (MObj)
+        MObj->releaseResources();
+    }
   }
 
 private:
   static std::atomic_uint MCounter;
   bool MModifyCounter;
-  std::unique_ptr<ResourceHandler> &MObj;
+  ResourceHandler *&MObj;
+  SpinLock &MObjProtector;
 };
 template <class ResourceHandler>
 std::atomic_uint ObjectUsageCounter<ResourceHandler>::MCounter{0};
 
-using LockGuard = std::lock_guard<SpinLock>;
-
 GlobalHandler::GlobalHandler() = default;
 GlobalHandler::~GlobalHandler() = default;
 
 GlobalHandler &GlobalHandler::instance() {
-  static GlobalHandler *SyclGlobalObjectsHandler = new GlobalHandler();
-  return *SyclGlobalObjectsHandler;
+  // No protection since sycl usage in parallel with main exit is not valid,
+  // otherwise MSyclGlobalObjectsHandler exists at any call to instance().
+  assert(MSyclGlobalObjectsHandler &&
+         "Handler must not be deallocated earlier");
+  return *MSyclGlobalObjectsHandler;
 }
 
 template <typename T, typename... Types>
@@ -94,8 +106,8 @@ Scheduler &GlobalHandler::getScheduler() {
 }
 
 void GlobalHandler::registerSchedulerUsage(bool ModifyCounter) {
-  thread_local ObjectUsageCounter<Scheduler> SchedulerCounter(MScheduler.Inst,
-                                                              ModifyCounter);
+  thread_local ObjectUsageCounter<GlobalHandler> SchedulerCounter(
+      MSyclGlobalObjectsHandler, MSyclGlobalHandlerProtector, ModifyCounter);
 }
 
 ProgramManager &GlobalHandler::getProgramManager() {
@@ -151,14 +163,14 @@ ThreadPool &GlobalHandler::getHostTaskThreadPool() {
 void GlobalHandler::releaseDefaultContexts() {
   // Release shared-pointers to SYCL objects.
 #ifndef _WIN32
-  GlobalHandler::instance().MPlatformToDefaultContextCache.Inst.reset(nullptr);
+  MPlatformToDefaultContextCache.Inst.reset(nullptr);
 #else
   // Windows does not maintain dependencies between dynamically loaded libraries
   // and can unload SYCL runtime dependencies before sycl.dll's DllMain has
   // finished. To avoid calls to nowhere, intentionally leak platform to device
   // cache. This will prevent destructors from being called, thus no PI cleanup
   // routines will be called in the end.
-  GlobalHandler::instance().MPlatformToDefaultContextCache.Inst.release();
+  MPlatformToDefaultContextCache.Inst.release();
 #endif
 }
 
@@ -178,8 +190,8 @@ void GlobalHandler::unloadPlugins() {
   // Call to GlobalHandler::instance().getPlugins() initializes plugins. If
   // user application has loaded SYCL runtime, and never called any APIs,
   // there's no need to load and unload plugins.
-  if (GlobalHandler::instance().MPlugins.Inst) {
-    for (plugin &Plugin : GlobalHandler::instance().getPlugins()) {
+  if (MPlugins.Inst) {
+    for (plugin &Plugin : getPlugins()) {
       // PluginParameter is reserved for future use that can control
       // some parameters in the plugin tear-down process.
       // Currently, it is not used.
@@ -189,43 +201,54 @@ void GlobalHandler::unloadPlugins() {
     }
   }
   // Clear after unload to avoid uses after unload.
-  GlobalHandler::instance().getPlugins().clear();
-}
-
-void GlobalHandler::drainThreadPool() {
-  if (MHostTaskThreadPool.Inst)
-    MHostTaskThreadPool.Inst->drain();
+  getPlugins().clear();
 }
 
 void shutdown() {
+  GlobalHandler *handler = nullptr;
+  {
+    const LockGuard Lock{GlobalHandler::MSyclGlobalHandlerProtector};
+    std::swap(handler, GlobalHandler::MSyclGlobalObjectsHandler);
+  }
+  assert(handler && "Handler could not be deallocated earlier");
   // Ensure neither host task is working so that no default context is accessed
   // upon its release
 
-  if (GlobalHandler::instance().MScheduler.Inst)
-    GlobalHandler::instance().MScheduler.Inst->releaseResources();
+  handler->releaseResources();
 
-  if (GlobalHandler::instance().MHostTaskThreadPool.Inst)
-    GlobalHandler::instance().MHostTaskThreadPool.Inst->finishAndWait();
+  if (handler->MHostTaskThreadPool.Inst)
+    handler->MHostTaskThreadPool.Inst->finishAndWait();
 
   // If default contexts are requested after the first default contexts have
   // been released there may be a new default context. These must be released
   // prior to closing the plugins.
   // Note: Releasing a default context here may cause failures in plugins with
   // global state as the global state may have been released.
-  GlobalHandler::instance().releaseDefaultContexts();
+  handler->releaseDefaultContexts();
 
   // First, release resources, that may access plugins.
-  GlobalHandler::instance().MPlatformCache.Inst.reset(nullptr);
-  GlobalHandler::instance().MScheduler.Inst.reset(nullptr);
-  GlobalHandler::instance().MProgramManager.Inst.reset(nullptr);
+  handler->MPlatformCache.Inst.reset(nullptr);
+  handler->MScheduler.Inst.reset(nullptr);
+  handler->MProgramManager.Inst.reset(nullptr);
 
   // Clear the plugins and reset the instance if it was there.
-  GlobalHandler::instance().unloadPlugins();
-  if (GlobalHandler::instance().MPlugins.Inst)
-    GlobalHandler::instance().MPlugins.Inst.reset(nullptr);
+  handler->unloadPlugins();
+  if (handler->MPlugins.Inst)
+    handler->MPlugins.Inst.reset(nullptr);
 
   // Release the rest of global resources.
-  delete &GlobalHandler::instance();
+  delete handler;
+}
+
+void GlobalHandler::drainThreadPool() {
+  if (MHostTaskThreadPool.Inst)
+    MHostTaskThreadPool.Inst->drain();
+}
+
+void GlobalHandler::releaseResources() {
+  drainThreadPool();
+  if (MScheduler.Inst)
+    MScheduler.Inst->releaseResources();
 }
 
 #ifdef _WIN32

sycl/source/detail/global_handler.hpp

@@ -76,12 +76,16 @@ class GlobalHandler {
 
   void unloadPlugins();
   void releaseDefaultContexts();
+
+  void releaseResources();
   void drainThreadPool();
 
   // For testing purposes only
   void attachScheduler(Scheduler *Scheduler);
 
 private:
+  static GlobalHandler *MSyclGlobalObjectsHandler;
+  static SpinLock MSyclGlobalHandlerProtector;
   friend void shutdown();
 
   // Constructor and destructor are declared out-of-line to allow incomplete

sycl/source/detail/scheduler/scheduler.cpp

@@ -392,11 +392,6 @@ Scheduler::~Scheduler() { DefaultHostQueue.reset(); }
 
 void Scheduler::releaseResources() {
 #ifndef _WIN32
-  if (DefaultHostQueue) {
-    DefaultHostQueue->wait();
-  }
-  GlobalHandler::instance().drainThreadPool();
-
   //  There might be some commands scheduled for post enqueue cleanup that
   //  haven't been freed because of the graph mutex being locked at the time,
   //  clean them up now.