[Pthreads] Fix worker.js in ES6 module environments (emscripten-core#21041)

This file can be an ES6 module if a package.json file indicates that all
files in the directory are.

We need to apply the same tricks as we apply to the main JS file in that
case, with createRequire etc.

We also need to emit the suffix .worker.mjs and not .js, or else node
will error.

Followup to emscripten-core#20939 and fixes the package.json discussion after that PR landed.

Author: kripken

ChangeLog.md

@@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works.
 
 3.1.52 (in development)
 -----------------------
+- Building with `pthreads+EXPORT_ES6` will now emit the worker file as
+  `NAME.worker.mjs` rather than `.js`. This is a necessary breaking change to
+  resolve other `pthreads+EXPORT_ES6` issues in Node.js (because Node.js is
+  affected by the suffix in some cases). (#21041)
 - Include paths added by ports (e.g. `-sUSE_SDL=2`) now use `-isystem` rather
   then `-I`.  This means that files in user-specified include directories will
   now take precedence over port includes. (#21014)

src/worker.js

@@ -18,6 +18,16 @@ var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions
 if (ENVIRONMENT_IS_NODE) {
   // Create as web-worker-like an environment as we can.
 
+  // See the parallel code in shell.js, but here we don't need the condition on
+  // multi-environment builds, as we do not have the need to interact with the
+  // modularization logic as shell.js must (see link.py:node_es6_imports and
+  // how that is used in link.py).
+#if EXPORT_ES6
+  const { createRequire } = await import('module');
+  /** @suppress{duplicate} */
+  var require = createRequire(import.meta.url);
+#endif
+
   var nodeWorkerThreads = require('worker_threads');
 
   var parentPort = nodeWorkerThreads.parentPort;
@@ -32,7 +42,13 @@ if (ENVIRONMENT_IS_NODE) {
     require,
     Module,
     location: {
+      // __filename is undefined in ES6 modules, and import.meta.url only in ES6
+      // modules.
+#if EXPORT_ES6
+      href: typeof __filename !== 'undefined' ? __filename : import.meta.url
+#else
       href: __filename
+#endif
     },
     Worker: nodeWorkerThreads.Worker,
     importScripts: (f) => vm.runInThisContext(fs.readFileSync(f, 'utf8'), {filename: f}),

test/test_other.py

@@ -343,10 +343,10 @@ def test_emcc_output_worker_mjs(self, args):
                       test_file('hello_world.c')] + args)
     src = read_file('subdir/hello_world.mjs')
     self.assertContained("new URL('hello_world.wasm', import.meta.url)", src)
-    self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src)
+    self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src)
     self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src)
     self.assertContained('export default Module;', src)
-    src = read_file('subdir/hello_world.worker.js')
+    src = read_file('subdir/hello_world.worker.mjs')
     self.assertContained("import('./hello_world.mjs')", src)
     self.assertContained('hello, world!', self.run_js('subdir/hello_world.mjs'))
 
@@ -358,7 +358,7 @@ def test_emcc_output_worker_mjs_single_file(self):
                       test_file('hello_world.c'), '-sSINGLE_FILE'])
     src = read_file('hello_world.mjs')
     self.assertNotContained("new URL('data:", src)
-    self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src)
+    self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src)
     self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src)
     self.assertContained('hello, world!', self.run_js('hello_world.mjs'))
 
@@ -400,11 +400,16 @@ def test_export_es6_allows_export_in_post_js(self):
     src = read_file('a.out.js')
     self.assertContained('export{doNothing};', src)
 
+  @parameterized({
+    '': (False,),
+    'package_json': (True,),
+  })
   @parameterized({
     '': ([],),
-    'pthreads': (['-pthread'],),
+    # load a worker before startup to check ES6 modules there as well
+    'pthreads': (['-pthread', '-sPTHREAD_POOL_SIZE=1'],),
   })
-  def test_export_es6(self, args):
+  def test_export_es6(self, args, package_json):
     self.run_process([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6',
                       '-o', 'hello.mjs'] + args)
     # In ES6 mode we use MODULARIZE, so we must instantiate an instance of the
@@ -413,6 +418,12 @@ def test_export_es6(self, args):
       import Hello from "./hello.mjs";
       Hello();
     ''')
+
+    if package_json:
+      # This makes node load all files in the directory as ES6 modules,
+      # including the worker.js file.
+      create_file('package.json', '{"type":"module"}')
+
     self.assertContained('hello, world!', self.run_js('runner.mjs'))
 
   def test_emcc_out_file(self):

tools/link.py

@@ -513,6 +513,10 @@ def do_split_module(wasm_file, options):
   building.run_binaryen_command('wasm-split', wasm_file + '.orig', outfile=wasm_file, args=args)
 
 
+def get_worker_js_suffix():
+  return '.worker.mjs' if settings.EXPORT_ES6 else '.worker.js'
+
+
 def setup_pthreads(target):
   if settings.RELOCATABLE:
     # phtreads + dyanmic linking has certain limitations
@@ -569,7 +573,7 @@ def setup_pthreads(target):
   building.user_requested_exports.update(worker_imports)
 
   # set location of worker.js
-  settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js'
+  settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + get_worker_js_suffix()
 
   if settings.MINIMAL_RUNTIME:
     building.user_requested_exports.add('exit')
@@ -1999,12 +2003,27 @@ def phase_memory_initializer(memfile):
   final_js += '.mem.js'
 
 
+# Unmangle previously mangled `import.meta` and `await import` references in
+# both main code and libraries.
+# See also: `preprocess` in parseTools.js.
+def fix_es6_import_statements(js_file):
+  if not settings.EXPORT_ES6 or not settings.USE_ES6_IMPORT_META:
+    return
+
+  src = read_file(js_file)
+  write_file(js_file, src
+             .replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
+             .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))
+
+
 def create_worker_file(input_file, target_dir, output_file):
   output_file = os.path.join(target_dir, output_file)
   input_file = utils.path_from_root(input_file)
   contents = shared.read_and_preprocess(input_file, expand_macros=True)
   write_file(output_file, contents)
 
+  fix_es6_import_statements(output_file)
+
   # Minify the worker JS file, if JS minification is enabled.
   if settings.MINIFY_WHITESPACE:
     contents = building.acorn_optimizer(output_file, ['minifyWhitespace'], return_output=True)
@@ -2045,17 +2064,8 @@ def phase_final_emitting(options, state, target, wasm_target, memfile):
     # mode)
     final_js = building.closure_compiler(final_js, advanced=False, extra_closure_args=options.closure_args)
 
-  # Unmangle previously mangled `import.meta` and `await import` references in
-  # both main code and libraries.
-  # See also: `preprocess` in parseTools.js.
-  if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META:
-    src = read_file(final_js)
-    final_js += '.esmeta.js'
-    write_file(final_js, src
-               .replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
-               .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))
-    shared.get_temp_files().note(final_js)
-    save_intermediate('es6-module')
+  fix_es6_import_statements(final_js)
+  save_intermediate('es6-module')
 
   # Apply pre and postjs files
   if options.extern_pre_js or options.extern_post_js:
@@ -2600,7 +2610,7 @@ def generate_worker_js(target, js_target, target_basename):
     proxy_worker_filename = get_subresource_location(js_target)
   else:
     # compiler output goes in .worker.js file
-    move_file(js_target, shared.replace_suffix(js_target, '.worker.js'))
+    move_file(js_target, shared.replace_suffix(js_target, get_worker_js_suffix()))
     worker_target_basename = target_basename + '.worker'
     proxy_worker_filename = (settings.PROXY_TO_WORKER_FILENAME or worker_target_basename) + '.js'