Use wasmImports as the single global symbol table for dynamic linking

Previously we were looking up symbols on the `Module` object which
meant building with `EXPORT_ALL` and exporting all symbols there.  This
change adds all needed symbols to the `wasmImports` maps and uses that
as the single source for symbol lookups.

This is split out from my work on #18376.

This should be a code size win in for most users of dynamic linking.  It
also avoids extra symbols being present on the `Module` object that the
user didn't ask for.

Author: sbc100

emcc.py

@@ -832,16 +832,14 @@ def process_dynamic_libs(dylibs, lib_dirs):
     # main module to avoid creating new invoke functions at runtime.
     imports = set(imports)
     imports = set(i for i in imports if not i.startswith('invoke_'))
-    weak_imports = sorted(imports.intersection(exports))
     strong_imports = sorted(imports.difference(exports))
     logger.debug('Adding symbols requirements from `%s`: %s', dylib, imports)
 
     mangled_imports = [shared.asmjs_mangle(e) for e in sorted(imports)]
     mangled_strong_imports = [shared.asmjs_mangle(e) for e in strong_imports]
     settings.SIDE_MODULE_IMPORTS.extend(mangled_imports)
-    settings.EXPORTED_FUNCTIONS.extend(mangled_strong_imports)
-    settings.EXPORT_IF_DEFINED.extend(weak_imports)
-    settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.extend(strong_imports)
+    settings.EXPORT_IF_DEFINED.extend(sorted(imports))
+    settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.extend(sorted(imports))
     building.user_requested_exports.update(mangled_strong_imports)
 
 
@@ -2023,7 +2021,6 @@ def phase_linker_setup(options, state, newargs):
 
   if settings.MAIN_MODULE == 1 or settings.SIDE_MODULE == 1:
     settings.LINKABLE = 1
-    settings.EXPORT_ALL = 1
 
   if settings.LINKABLE and settings.USER_EXPORTED_FUNCTIONS:
     diagnostics.warning('unused-command-line-argument', 'EXPORTED_FUNCTIONS is not valid with LINKABLE set (normally due to SIDE_MODULE=1/MAIN_MODULE=1) since all functions are exported this mode.  To export only a subset use SIDE_MODULE=2/MAIN_MODULE=2')

emscripten.py

@@ -424,15 +424,14 @@ def emscript(in_wasm, out_wasm, outfile_js, memfile):
     out.write(normalize_line_endings(pre))
     pre = None
 
-    sending = create_sending(invoke_funcs, metadata)
     receiving = create_receiving(exports)
 
     if settings.MINIMAL_RUNTIME:
       if settings.DECLARE_ASM_MODULE_EXPORTS:
         post = compute_minimal_runtime_initializer_and_exports(post, exports, receiving)
       receiving = ''
 
-    module = create_module(sending, receiving, invoke_funcs, metadata)
+    module = create_module(receiving, invoke_funcs, metadata, forwarded_json['librarySymbols'])
 
     write_output_file(out, module)
 
@@ -671,7 +670,7 @@ def add_standard_wasm_imports(send_items_map):
     send_items_map[s] = s
 
 
-def create_sending(invoke_funcs, metadata):
+def create_sending(invoke_funcs, metadata, library_symbols):
   # Map of wasm imports to mangled/external/JS names
   send_items_map = {}
 
@@ -685,6 +684,28 @@ def create_sending(invoke_funcs, metadata):
 
   add_standard_wasm_imports(send_items_map)
 
+  if settings.MAIN_MODULE:
+    # When including dynamic linking support, also add any JS library functions
+    # that are part of EXPORTED_FUNCTIONS (or in the case of MAIN_MODULE=1 add
+    # all JS library functions).  This allows `dlsym(RTLD_DEFAULT)` to lookup JS
+    # library functions, since `wasmImports` acts as the global symbol table.
+    wasm_exports = set(metadata.exports)
+    library_symbols = set(library_symbols)
+    if settings.MAIN_MODULE == 1:
+      for f in library_symbols:
+        if shared.is_c_symbol(f):
+          demangled = shared.demangle_c_symbol_name(f)
+          if demangled in wasm_exports:
+            continue
+          send_items_map[demangled] = f
+    else:
+      for f in settings.EXPORTED_FUNCTIONS + settings.SIDE_MODULE_IMPORTS:
+        if f in library_symbols and shared.is_c_symbol(f):
+          demangled = shared.demangle_c_symbol_name(f)
+          if demangled in wasm_exports:
+            continue
+          send_items_map[demangled] = f
+
   sorted_keys = sorted(send_items_map.keys())
   return '{\n  ' + ',\n  '.join('"' + k + '": ' + send_items_map[k] for k in sorted_keys) + '\n}'
 
@@ -720,10 +741,10 @@ def make_export_wrappers(exports, delay_assignment):
     elif delay_assignment:
       # With assertions disabled the wrapper will replace the global var and Module var on
       # first use.
-      wrapper += '''function() {
-  return (%(mangled)s = %(exported)sModule["asm"]["%(name)s"]).apply(null, arguments);
-};
-''' % {'mangled': mangled, 'name': name, 'exported': exported}
+      wrapper += f'''function() {{
+  return ({mangled} = {exported}Module["asm"]["{name}"]).apply(null, arguments);
+}};
+'''
     else:
       wrapper += 'asm["%s"]' % name
 
@@ -771,11 +792,12 @@ def create_receiving(exports):
     return '\n'.join(receiving) + '\n'
 
 
-def create_module(sending, receiving, invoke_funcs, metadata):
+def create_module(receiving, invoke_funcs, metadata, library_symbols):
   invoke_wrappers = create_invoke_wrappers(invoke_funcs)
   receiving += create_named_globals(metadata)
   module = []
 
+  sending = create_sending(invoke_funcs, metadata, library_symbols)
   module.append('var wasmImports = %s;\n' % sending)
   if settings.ASYNCIFY and (settings.ASSERTIONS or settings.ASYNCIFY == 2):
     # instrumenting imports is used in asyncify in two ways: to add assertions

src/jsifier.js

@@ -271,10 +271,19 @@ function ${name}(${args}) {
           // (not useful to warn/error multiple times)
           LibraryManager.library[symbol + '__docs'] = '/** @type {function(...*):?} */';
         } else {
-          const target = `Module['${mangled}']`;
+          // Create a stub for this symbol which can later be replaced by the
+          // dynamic linker.  If this stub is called before the symbol is
+          // resolved assert in debug builds or trap in release builds.
+          if (ASYNCIFY) {
+            // See the definition of asyncifyStubs in preamble.js for why this
+            // is needed.
+            target = `asyncifyStubs['${symbol}']`;
+          } else {
+            target = `wasmImports['${symbol}']`;
+          }
           let assertion = '';
           if (ASSERTIONS) {
-            assertion += `if (!${target}) abort("external symbol '${symbol}' is missing. perhaps a side module was not linked in? if this function was expected to arrive from a system library, try to build the MAIN_MODULE with EMCC_FORCE_STDLIBS=1 in the environment");\n`;
+            assertion += `if (!${target} || ${target}.stub) abort("external symbol '${symbol}' is missing. perhaps a side module was not linked in? if this function was expected to arrive from a system library, try to build the MAIN_MODULE with EMCC_FORCE_STDLIBS=1 in the environment");\n`;
           }
           const functionBody = assertion + `return ${target}.apply(null, arguments);`;
           LibraryManager.library[symbol] = new Function(functionBody);
@@ -426,6 +435,9 @@ function ${name}(${args}) {
       }
       if (isStub) {
         contentText += `\n${mangled}.stub = true;`;
+        if (ASYNCIFY) {
+          contentText += `\nasyncifyStubs['${symbol}'] = undefined;`;
+        }
       }
 
       let commentText = '';

src/library_dylink.js

@@ -10,40 +10,44 @@ var dlopenMissingError = "'To use dlopen, you need enable dynamic linking, see h
 
 var LibraryDylink = {
 #if RELOCATABLE
+  $isSymbolDefined: function(symName) {
+    // Ignore 'stub' symbols that are auto-generated as part of the original
+    // `wasmImports` used to instantate the main module.
+    var existing = wasmImports[symName];
+    if (!existing || existing.stub) {
+      return false;
+    }
+#if ASYNCIFY
+    // Even if a symbol exists in wasmImports, and is not itself a stub, it
+    // could be an ASYNCIFY wrapper function that wraps a stub function.
+    if (symName in asyncifyStubs && !asyncifyStubs[symName]) {
+      return false;
+    }
+#endif
+    return true;
+  },
+
+  $resolveGlobalSymbol__deps: ['$isSymbolDefined'],
   $resolveGlobalSymbol__internal: true,
-  $resolveGlobalSymbol__deps: ['$asmjsMangle'],
-  $resolveGlobalSymbol: function(symName, direct) {
+  $resolveGlobalSymbol: function(symName, direct = false) {
     var sym;
 #if !WASM_BIGINT
     if (direct) {
       // First look for the orig$ symbol which is the symbols without
       // any legalization performed.
       sym = wasmImports['orig$' + symName];
+      if (sym) return sym;
     }
 #endif
-    if (!sym) {
+    if (isSymbolDefined(symName)) {
       sym = wasmImports[symName];
-      // Ignore 'stub' symbols that are auto-generated as part of the original
-      // `wasmImports` used to instantate the main module.
-      if (sym && sym.stub) sym = undefined;
+    } else if (symName.startsWith('invoke_')) {
+      // Create (and cache) new invoke_ functions on demand.
+      sym = wasmImports[symName] = createInvokeFunction(symName.split('_')[1]);
     }
-
-    // Check for the symbol on the Module object.  This is the only
-    // way to dynamically access JS library symbols that were not
-    // referenced by the main module (and therefore not part of the
-    // initial set of symbols included in wasmImports when it
-    // was declared.
-    if (!sym) {
-      sym = Module[asmjsMangle(symName)];
-    }
-
-    if (!sym && symName.startsWith('invoke_')) {
-      sym = createInvokeFunction(symName.split('_')[1]);
-    }
-
 #if !DISABLE_EXCEPTION_CATCHING
-    if (!sym && symName.startsWith("__cxa_find_matching_catch")) {
-      sym = Module["___cxa_find_matching_catch"];
+    else if (symName.startsWith('__cxa_find_matching_catch_')) {
+      sym = wasmImports['__cxa_find_matching_catch'];
     }
 #endif
     return sym;
@@ -456,24 +460,15 @@ var LibraryDylink = {
   },
 
   // Module.symbols <- libModule.symbols (flags.global handler)
-  $mergeLibSymbols__deps: ['$asmjsMangle'],
+  $mergeLibSymbols__deps: ['$asmjsMangle', '$isSymbolDefined'],
   $mergeLibSymbols: function(exports, libName) {
     // add symbols into global namespace TODO: weak linking etc.
     for (var sym in exports) {
       if (!exports.hasOwnProperty(sym)) {
         continue;
       }
-
-      // When RTLD_GLOBAL is enable, the symbols defined by this shared object will be made
-      // available for symbol resolution of subsequently loaded shared objects.
-      //
-      // We should copy the symbols (which include methods and variables) from SIDE_MODULE to MAIN_MODULE.
-
-      if (!wasmImports.hasOwnProperty(sym)) {
-        wasmImports[sym] = exports[sym];
-      }
 #if ASSERTIONS == 2
-      else {
+      if (isSymbolDefined(sym)) {
         var curr = wasmImports[sym], next = exports[sym];
         // don't warn on functions - might be odr, linkonce_odr, etc.
         if (!(typeof curr == 'function' && typeof next == 'function')) {
@@ -482,19 +477,40 @@ var LibraryDylink = {
       }
 #endif
 
-      // Export native export on the Module object.
-      // TODO(sbc): Do all users want this?  Should we skip this by default?
-      var module_sym = asmjsMangle(sym);
-      if (!Module.hasOwnProperty(module_sym)) {
-        Module[module_sym] = exports[sym];
+      // When RTLD_GLOBAL is enabled, the symbols defined by this shared object
+      // will be made available for symbol resolution of subsequently loaded
+      // shared objects.
+      //
+      // We should copy the symbols (which include methods and variables) from
+      // SIDE_MODULE to MAIN_MODULE.
+      const setImport = (target) => {
+#if ASYNCIFY
+        if (target in asyncifyStubs) {
+          asyncifyStubs[target] = exports[sym]
+        }
+#endif
+        if (!isSymbolDefined(target)) {
+          wasmImports[target] = exports[sym];
+        }
       }
+      setImport(sym);
+
 #if !hasExportedSymbol('main')
-      // If the main module doesn't define main it could be defined in one of
-      // the side modules, and we need to handle the mangled named.
-      if (sym == '__main_argc_argv') {
-        Module['_main'] = exports[sym];
+      // Special case for handling of main symbol:  If a side module exports
+      // `main` that also acts a definition for `__main_argc_argv` and vice
+      // versa.
+      const main_alias = '__main_argc_argv';
+      if (sym == 'main') {
+        setImport(main_alias)
+      }
+      if (sym == main_alias) {
+        setImport('main')
       }
 #endif
+
+      if (sym.startsWith('dynCall_') && !Module.hasOwnProperty(sym)) {
+        Module[sym] = exports[sym];
+      }
     }
   },
 
@@ -575,7 +591,7 @@ var LibraryDylink = {
       var moduleExports;
 
       function resolveSymbol(sym) {
-        var resolved = resolveGlobalSymbol(sym, false);
+        var resolved = resolveGlobalSymbol(sym);
         if (!resolved) {
           resolved = moduleExports[sym];
         }
@@ -613,7 +629,7 @@ var LibraryDylink = {
               return tableBase;
 #endif
           }
-          if (prop in wasmImports) {
+          if (prop in wasmImports && !wasmImports[prop].stub) {
             // No stub needed, symbol already exists in symbol table
             return wasmImports[prop];
           }
@@ -800,7 +816,7 @@ var LibraryDylink = {
   // If a library was already loaded, it is not loaded a second time. However
   // flags.global and flags.nodelete are handled every time a load request is made.
   // Once a library becomes "global" or "nodelete", it cannot be removed or unloaded.
-  $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', '$asmjsMangle', '$isInternalSym', '$mergeLibSymbols'],
+  $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', '$isInternalSym', '$mergeLibSymbols'],
   $loadDynamicLibrary__docs: '/** @param {number=} handle */',
   $loadDynamicLibrary: function(lib, flags = {global: true, nodelete: true}, handle = 0) {
 #if DYLINK_DEBUG

src/parseTools.js

@@ -1100,6 +1100,25 @@ function getUnsharedTextDecoderView(heap, start, end) {
   return `${heap}.buffer instanceof SharedArrayBuffer ? ${shared} : ${unshared}`;
 }
 
+function getEntryFunction() {
+  var entryFunction = 'main';
+  if (STANDALONE_WASM) {
+    if (EXPECT_MAIN) {
+      entryFunction = '_start';
+    } else {
+      entryFunction = '_initialize';
+    }
+  } else if (PROXY_TO_PTHREAD) {
+    // User requested the PROXY_TO_PTHREAD option, so call a stub main which pthread_create()s a new thread
+    // that will call the user's real main() for the application.
+    entryFunction = '_emscripten_proxy_main';
+  }
+  if (MAIN_MODULE) {
+    return `resolveGlobalSymbol('${entryFunction}');`
+  }
+  return '_' + entryFunction;
+}
+
 function preJS() {
   let result = '';
   for (const fileName of PRE_JS_FILES) {

src/postamble.js

@@ -31,21 +31,7 @@ function callMain() {
   assert(__ATPRERUN__.length == 0, 'cannot call main when preRun functions remain to be called');
 #endif
 
-#if STANDALONE_WASM
-#if EXPECT_MAIN
-  var entryFunction = Module['__start'];
-#else
-  var entryFunction = Module['__initialize'];
-#endif
-#else
-#if PROXY_TO_PTHREAD
-  // User requested the PROXY_TO_PTHREAD option, so call a stub main which pthread_create()s a new thread
-  // that will call the user's real main() for the application.
-  var entryFunction = Module['__emscripten_proxy_main'];
-#else
-  var entryFunction = Module['_main'];
-#endif
-#endif
+  var entryFunction = {{{ getEntryFunction() }}};
 
 #if MAIN_MODULE
   // Main modules can't tell if they have main() at compile time, since it may

src/preamble.js

@@ -1223,4 +1223,13 @@ function runMemoryInitializer() {
 }
 #endif // MEM_INIT_IN_WASM == 0
 
+#if MAIN_MODULE && ASYNCIFY
+// With MAIN_MODULE + ASYNCIFY the normal method of placing stub functions in
+// wasmImports for as-yet-undefined symbols doesn't work since ASYNCIFY then
+// wraps these stub functions and we can't then replace them directly.  Instead
+// the stub functions call into `asyncifyStubs` which gets populated by the
+// dynamic linker as symbols are loaded.
+var asyncifyStubs = {};
+#endif
+
 // === Body ===

test/other/metadce/test_metadce_cxx_ctors1.jssize

@@ -1 +1 @@
-26008
+26013

test/other/metadce/test_metadce_cxx_ctors2.jssize

@@ -1 +1 @@
-25972
+25977

test/other/metadce/test_metadce_cxx_except.jssize

@@ -1 +1 @@
-30545
+30550