Block in dlopen until all threads have loaded the module

Fixes: #18345

Author: sbc100

emcc.py

@@ -1645,7 +1645,13 @@ def setup_pthreads(target):
   ]
 
   if settings.MAIN_MODULE:
-    settings.REQUIRED_EXPORTS += ['_emscripten_thread_sync_code', '__dl_seterr']
+    settings.REQUIRED_EXPORTS += [
+      '_emscripten_dlsync_self',
+      '_emscripten_dlsync_self_async',
+      '_emscripten_proxy_dlsync',
+      '_emscripten_proxy_dlsync_async',
+      '__dl_seterr',
+    ]
 
   settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [
     '$exitOnMainThread',

src/library_pthread.js

@@ -31,6 +31,9 @@ var LibraryPThread = {
   $PThread__deps: ['_emscripten_thread_init',
                    '$killThread',
                    '$cancelThread', '$cleanupThread', '$zeroMemory',
+#if MAIN_MODULE
+                   '$markAsZombie',
+#endif
                    '$spawnThread',
                    '_emscripten_thread_free_data',
                    'exit',
@@ -55,6 +58,12 @@ var LibraryPThread = {
     // the reverse mapping, each worker has a `pthread_ptr` when its running a
     // pthread.
     pthreads: {},
+#if MAIN_MODULE
+    // Zombie threads are threads that have finished running but we not yet
+    // joined.
+    zombieThreads: {},
+    outstandingPromises: {},
+#endif
 #if PTHREADS_DEBUG
     nextWorkerID: 1,
     debugInit: function() {
@@ -267,6 +276,10 @@ var LibraryPThread = {
           spawnThread(d);
         } else if (cmd === 'cleanupThread') {
           cleanupThread(d['thread']);
+#if MAIN_MODULE
+        } else if (cmd === 'markAsZombie') {
+          markAsZombie(d['thread']);
+#endif
         } else if (cmd === 'killThread') {
           killThread(d['thread']);
         } else if (cmd === 'cancelThread') {
@@ -538,11 +551,20 @@ var LibraryPThread = {
   },
 
   $cleanupThread: function(pthread_ptr) {
+#if PTHREADS_DEBUG
+    dbg('cleanupThread: ' + ptrToString(pthread_ptr))
+#endif
 #if ASSERTIONS
     assert(!ENVIRONMENT_IS_PTHREAD, 'Internal Error! cleanupThread() can only ever be called from main application thread!');
     assert(pthread_ptr, 'Internal Error! Null pthread_ptr in cleanupThread!');
 #endif
     var worker = PThread.pthreads[pthread_ptr];
+#if MAIN_MODULE
+    delete PThread.zombieThreads[pthread_ptr];
+    if (pthread_ptr in PThread.outstandingPromises) {
+      PThread.outstandingPromises[pthread_ptr].resolve();
+    }
+#endif
     assert(worker);
     PThread.returnWorkerToPool(worker);
   },
@@ -1044,7 +1066,7 @@ var LibraryPThread = {
     // Before we call the thread entry point, make sure any shared libraries
     // have been loaded on this there.  Otherwise our table migth be not be
     // in sync and might not contain the function pointer `ptr` at all.
-    __emscripten_thread_sync_code();
+    __emscripten_dlsync_self();
 #endif
     // pthread entry points are always of signature 'void *ThreadMain(void *arg)'
     // Native codebases sometimes spawn threads with other thread entry point
@@ -1073,6 +1095,90 @@ var LibraryPThread = {
 #endif
   },
 
+#if MAIN_MODULE
+  _emscripten_thread_exit_joinable: function(thread) {
+    // Called when a thread exits and is joinable.  This puts the thread
+    // into zombie state where it can't run anymore work but cannot yet
+    // be cleaned up.
+    if (!ENVIRONMENT_IS_PTHREAD) markAsZombie(thread);
+    else postMessage({ 'cmd': 'markAsZombie', 'thread': thread });
+  },
+
+  $markAsZombie: function(pthread_ptr) {
+#if PTHREADS_DEBUG
+    dbg('markAsZombie: ' + ptrToString(pthread_ptr));
+#endif
+    PThread.zombieThreads[pthread_ptr] = true;
+    if (pthread_ptr in PThread.outstandingPromises) {
+      PThread.outstandingPromises[pthread_ptr].resolve();
+    }
+  },
+
+  // Asynchronous version dlsync_threads. Always run on the main thread.
+  // This work happens asynchronously. The `callback` is called once this work
+  // is completed, passing the ctx.
+  // TODO(sbc): Should we make a new form of __proxy attribute for JS library
+  // function that run asynchronously like but blocks the caller until they are
+  // done.  Perhaps "sync_with_ctx"?
+  _emscripten_dlsync_threads_async__sig: 'viii',
+  _emscripten_dlsync_threads_async__deps: ['_emscripten_proxy_dlsync_async', '$newNativePromise'],
+  _emscripten_dlsync_threads_async: function(caller, callback, ctx) {
+#if PTHREADS_DEBUG
+    dbg("_emscripten_dlsync_threads_async caller=" + ptrToString(caller));
+#endif
+#if ASSERTIONS
+    assert(!ENVIRONMENT_IS_PTHREAD, 'Internal Error! _emscripten_dlsync_threads_async() can only ever be called from main thread');
+#endif
+
+    const promises = [];
+    assert(Object.keys(PThread.outstandingPromises).length === 0);
+
+    // This first promise resolves once the main thread has loaded all modules.
+    promises.push(newNativePromise(__emscripten_dlsync_self_async, null).promise);
+
+    // We then create a sequence of promises, one per thread, that resolve once
+    // each thread has performed its sync using _emscripten_proxy_dlsync.
+    // Any new threads that are created after this call will automaticaly be
+    // in sync because we call `__emscripten_dlsync_self` in
+    // invokeEntryPoint before the threads entry point is called.
+    for (const ptr of Object.keys(PThread.pthreads)) {
+      const pthread_ptr = Number(ptr);
+      if (pthread_ptr !== caller && !(pthread_ptr in PThread.zombieThreads)) {
+        var p = newNativePromise(__emscripten_proxy_dlsync_async, pthread_ptr);
+        PThread.outstandingPromises[pthread_ptr] = p;
+        promises.push(p.promise);
+      }
+    }
+
+#if PTHREADS_DEBUG
+    dbg('_emscripten_dlsync_threads_async: waiting on ' + promises.length + ' promises');
+#endif
+    // Once all promises are resolved then we know all threads are in sync and
+    // we can call the callback.
+    Promise.all(promises).then(() => {
+      PThread.outstandingPromises = {};
+#if PTHREADS_DEBUG
+      dbg('_emscripten_dlsync_threads_async done: calling callback');
+#endif
+      {{{ makeDynCall('vp', 'callback') }}}(ctx);
+    });
+  },
+
+  // Synchronous version dlsync_threads. Always run on the main thread.
+  _emscripten_dlsync_threads__deps: ['_emscripten_proxy_dlsync'],
+  _emscripten_dlsync_threads: function() {
+#if ASSERTIONS
+    assert(!ENVIRONMENT_IS_PTHREAD, 'Internal Error! _emscripten_dlsync_threads() can only ever be called from main thread');
+#endif
+    for (const ptr of Object.keys(PThread.pthreads)) {
+      const pthread_ptr = Number(ptr);
+      if (!(pthread_ptr in PThread.zombieThreads)) {
+        __emscripten_proxy_dlsync(pthread_ptr);
+      }
+    }
+  },
+#endif // MAIN_MODULE
+
   $executeNotifiedProxyingQueue: function(queue) {
     // Set the notification state to processing.
     Atomics.store(HEAP32, queue >> 2, {{{ cDefine('NOTIFICATION_RECEIVED') }}});

system/include/emscripten/threading.h

@@ -275,9 +275,6 @@ int emscripten_pthread_attr_settransferredcanvases(pthread_attr_t *a, const char
 // blocking is not enabled, see ALLOW_BLOCKING_ON_MAIN_THREAD.
 void emscripten_check_blocking_allowed(void);
 
-// Experimental API for syncing loaded code between pthreads.
-void _emscripten_thread_sync_code();
-
 #ifdef __cplusplus
 }
 #endif