From c4f2cf9fd25e76e61712dd1143ced08cfc1d9530 Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Tue, 17 Dec 2019 20:34:11 -0600 Subject: [PATCH] module: move esm loader hooks to worker thread --- lib/internal/bootstrap/pre_execution.js | 38 +++- lib/internal/main/worker_thread.js | 25 ++- lib/internal/modules/esm/ipc_types.js | 99 ++++++++++ lib/internal/modules/esm/loader.js | 140 +++++++------- lib/internal/modules/esm/translators.js | 22 +-- lib/internal/modules/esm/worker.js | 180 ++++++++++++++++++ lib/internal/modules/run_main.js | 11 +- lib/internal/process/esm_loader.js | 29 +-- lib/internal/worker.js | 29 ++- node.gyp | 2 + test/es-module/test-esm-loader-isolation.mjs | 37 ++++ .../builtin-named-exports-loader.mjs | 4 +- .../es-module-loaders/isolation-hook.mjs | 38 ++++ .../es-module-loaders/loader-with-dep.mjs | 4 +- .../esm_display_syntax_error_import.out | 1 + ...esm_display_syntax_error_import_module.out | 1 + test/message/esm_loader_not_found.out | 26 ++- .../esm_loader_not_found_cjs_hint_bare.out | 3 +- ...esm_loader_not_found_cjs_hint_relative.out | 25 ++- test/message/esm_loader_syntax_error.out | 14 +- test/parallel/test-bootstrap-modules.js | 2 + 21 files changed, 574 insertions(+), 156 deletions(-) create mode 100644 lib/internal/modules/esm/ipc_types.js create mode 100644 lib/internal/modules/esm/worker.js create mode 100644 test/es-module/test-esm-loader-isolation.mjs create mode 100644 test/fixtures/es-module-loaders/isolation-hook.mjs diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index f60814d2dc9e28..8ea59ee56eb9be 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -28,6 +28,8 @@ function prepareMainThreadExecution(expandArgv1 = false) { setupDebugEnv(); + // Load policy from disk and parse it. + initializePolicy(); // Print stack trace on `SIGINT` if option `--trace-sigint` presents. setupStacktracePrinterOnSigint(); @@ -45,9 +47,6 @@ function prepareMainThreadExecution(expandArgv1 = false) { // process.disconnect(). setupChildProcessIpcChannel(); - // Load policy from disk and parse it. - initializePolicy(); - // If this is a worker in cluster mode, start up the communication // channel. This needs to be done before any user code gets executed // (including preload modules). @@ -56,7 +55,32 @@ function prepareMainThreadExecution(expandArgv1 = false) { initializeDeprecations(); initializeWASI(); initializeCJSLoader(); - initializeESMLoader(); + + function startLoaders() { + const loaderHREF = getOptionValue('--experimental-loader'); + if (!loaderHREF) return null; + const { InternalWorker } = require('internal/worker'); + const { MessageChannel } = require('internal/worker/io'); + // DO NOT ADD CODE ABOVE THIS LINE + // THIS SHOULD POST THE PORTS ASAP TO THE PARENT + const { + port1: outsideBelowPort, + port2: insideBelowPort + } = new MessageChannel(); + outsideBelowPort.name = 'outsideBelowPort'; + InternalWorker('internal/modules/esm/worker', { + // stdout: true, + // strerr: true, + transferList: [ insideBelowPort ], + workerData: { + loaderHREF, + insideBelowPort, + insideAbovePort: null + } + }).unref(); + return outsideBelowPort; + } + initializeESMLoader(startLoaders()); const CJSLoader = require('internal/modules/cjs/loader'); assert(!CJSLoader.hasLoadedAnyUserCJSModule); @@ -332,12 +356,12 @@ function initializeClusterIPC() { } } +const { pathToFileURL, URL } = require('url'); function initializePolicy() { const experimentalPolicy = getOptionValue('--experimental-policy'); if (experimentalPolicy) { process.emitWarning('Policies are experimental.', 'ExperimentalWarning'); - const { pathToFileURL, URL } = require('url'); // URL here as it is slightly different parsing // no bare specifiers for now let manifestURL; @@ -396,7 +420,7 @@ function initializeCJSLoader() { require('internal/modules/run_main').executeUserEntryPoint; } -function initializeESMLoader() { +function initializeESMLoader(bottomLoader) { // Create this WeakMap in js-land because V8 has no C++ API for WeakMap. internalBinding('module_wrap').callbackMap = new SafeWeakMap(); @@ -404,6 +428,8 @@ function initializeESMLoader() { setImportModuleDynamicallyCallback, setInitializeImportMetaObjectCallback } = internalBinding('module_wrap'); + const esmLoader = require('internal/modules/esm/loader'); + esmLoader.initUserLoaders(bottomLoader); const esm = require('internal/process/esm_loader'); // Setup per-isolate callbacks that locate data or callbacks that we keep // track of for different ESM modules. diff --git a/lib/internal/main/worker_thread.js b/lib/internal/main/worker_thread.js index 6e3935d1382965..e6ae3952f3a5c8 100644 --- a/lib/internal/main/worker_thread.js +++ b/lib/internal/main/worker_thread.js @@ -98,7 +98,9 @@ port.on('message', (message) => { cwdCounter, filename, doEval, + internal, workerData, + loaderPort, publicPort, manifestSrc, manifestURL, @@ -113,14 +115,16 @@ port.on('message', (message) => { initializeDeprecations(); initializeWASI(); initializeCJSLoader(); - initializeESMLoader(); - - const CJSLoader = require('internal/modules/cjs/loader'); - assert(!CJSLoader.hasLoadedAnyUserCJSModule); - loadPreloadModules(); - initializeFrozenIntrinsics(); - if (argv !== undefined) { - process.argv = process.argv.concat(argv); + initializeESMLoader(loaderPort); + + if (!internal) { + const CJSLoader = require('internal/modules/cjs/loader'); + assert(!CJSLoader.hasLoadedAnyUserCJSModule); + loadPreloadModules(); + initializeFrozenIntrinsics(); + if (argv !== undefined) { + process.argv = process.argv.concat(argv); + } } publicWorker.parentPort = publicPort; publicWorker.workerData = workerData; @@ -147,7 +151,9 @@ port.on('message', (message) => { debug(`[${threadId}] starts worker script ${filename} ` + `(eval = ${eval}) at cwd = ${process.cwd()}`); port.postMessage({ type: UP_AND_RUNNING }); - if (doEval) { + if (internal) { + require(filename); + } else if (doEval) { const { evalScript } = require('internal/process/execution'); const name = '[worker eval]'; // This is necessary for CJS module compilation. @@ -164,6 +170,7 @@ port.on('message', (message) => { // runMain here might be monkey-patched by users in --require. // XXX: the monkey-patchability here should probably be deprecated. process.argv.splice(1, 0, filename); + const CJSLoader = require('internal/modules/cjs/loader'); CJSLoader.Module.runMain(filename); } } else if (message.type === STDIO_PAYLOAD) { diff --git a/lib/internal/modules/esm/ipc_types.js b/lib/internal/modules/esm/ipc_types.js new file mode 100644 index 00000000000000..21da9a005cad54 --- /dev/null +++ b/lib/internal/modules/esm/ipc_types.js @@ -0,0 +1,99 @@ +/* eslint-disable */ +let nextId = 1; +function getNewId() { + const id = nextId; + nextId++; + return id; +} + +class LoaderWorker { + resolveImportURL() { + throw new TypeError('Not implemented'); + } + + getFormat() { + throw new TypeError('Not implemented'); + } + + getSource() { + throw new TypeError('Not implemented'); + } + + transformSource() { + throw new TypeError('Not implemented'); + } + + addBelowPort() { + throw new TypeError('Not implemented'); + } +} + +class RemoteLoaderWorker { + constructor(port) { + this._port = port; + this._pending = new Map(); + + this._port.on('message', this._handleMessage.bind(this)); + this._port.unref(); + } + + _handleMessage({ id, error, result }) { + if (!this._pending.has(id)) return; + const { resolve, reject } = this._pending.get(id); + this._pending.delete(id); + this._port.unref(); + + if (error) { + reject(error); + } else { + resolve(result); + } + } + + _send(method, params, transferList) { + const id = getNewId(); + const message = { id, method, params }; + return new Promise((resolve, reject) => { + this._pending.set(id, { resolve, reject }); + this._port.postMessage(message, transferList); + this._port.ref(); + }); + } +} + +for (const method of Object.getOwnPropertyNames(LoaderWorker.prototype)) { + Object.defineProperty(RemoteLoaderWorker.prototype, method, { + configurable: true, + enumerable: false, + value(data, transferList) { + return this._send(method, data, transferList); + }, + }); +} + +function connectIncoming(port, instance) { + port.on('message', async ({ id, method, params }) => { + if (!id) return; + + let result = null; + let error = null; + try { + if (typeof instance[method] !== 'function') { + throw new TypeError(`No such RPC method: ${method}`); + } + result = await instance[method](params); + } catch (e) { + error = e; + } + port.postMessage({ id, error, result }); + }); + + return instance; +} + +module.exports = { + connectIncoming, + getNewId, + LoaderWorker, + RemoteLoaderWorker, +}; diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index de08845a820d77..f54685a2e9ec46 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -1,11 +1,15 @@ 'use strict'; const { - FunctionPrototypeBind, ObjectSetPrototypeOf, SafeMap, } = primordials; +const { + threadId +} = internalBinding('worker'); + +const assert = require('internal/assert'); const { ERR_INVALID_ARG_VALUE, ERR_INVALID_RETURN_PROPERTY, @@ -23,14 +27,31 @@ const { DEFAULT_CONDITIONS, } = require('internal/modules/esm/resolve'); const { defaultGetFormat } = require('internal/modules/esm/get_format'); -const { defaultGetSource } = require( - 'internal/modules/esm/get_source'); +const { defaultGetSource } = require('internal/modules/esm/get_source'); const { defaultTransformSource } = require( 'internal/modules/esm/transform_source'); const { translators } = require( 'internal/modules/esm/translators'); const { getOptionValue } = require('internal/options'); +const defaultLoaderWorker = { + resolveImportURL({ specifier, base, conditions }) { + return defaultResolve(specifier, { parentURL: base, conditions }); + }, + + getFormat({ url }) { + return defaultGetFormat(url); + }, + + getSource({ url, format }) { + return defaultGetSource(url, { format }); + }, + + transformSource({ source, url, format }) { + return defaultTransformSource(source, { url, format }); + }, +}; + /* A Loader instance is used as the main entry point for loading ES modules. * Currently, this is a singleton -- there is only one used for loading * the main module and everything in its dependency graph. */ @@ -46,41 +67,34 @@ class Loader { // Map of already-loaded CJS modules to use this.cjsCache = new SafeMap(); - // This hook is called before the first root module is imported. It's a - // function that returns a piece of code that runs as a sloppy-mode script. - // The script may evaluate to a function that can be called with a - // `getBuiltin` helper that can be used to retrieve builtins. - // If the hook returns `null` instead of a source string, it opts out of - // running any preload code. - // The preload code runs as soon as the hook module has finished evaluating. - this._getGlobalPreloadCode = null; - // The resolver has the signature - // (specifier : string, parentURL : string, defaultResolve) - // -> Promise<{ url : string }> - // where defaultResolve is ModuleRequest.resolve (having the same - // signature itself). - this._resolve = defaultResolve; - // This hook is called after the module is resolved but before a translator - // is chosen to load it; the format returned by this function is the name - // of a translator. - this._getFormat = defaultGetFormat; - // This hook is called just before the source code of an ES module file - // is loaded. - this._getSource = defaultGetSource; - // This hook is called just after the source code of an ES module file - // is loaded, but before anything is done with the string. - this._transformSource = defaultTransformSource; // The index for assigning unique URLs to anonymous module evaluation this.evalIndex = 0; } + _getSource(params) { + const bottomLoaderRPC = exports.getBottomLoaderRPC(); + return bottomLoaderRPC.getSource(params); + } + + _transformSource(params) { + const bottomLoaderRPC = exports.getBottomLoaderRPC(); + return bottomLoaderRPC.transformSource(params); + } + async resolve(specifier, parentURL) { const isMain = parentURL === undefined; if (!isMain) validateString(parentURL, 'parentURL'); + validateString(specifier, 'specifier'); + + const bottomLoaderRPC = exports.getBottomLoaderRPC(); + const resolveResponse = await bottomLoaderRPC.resolveImportURL({ + clientId: threadId, + specifier, + base: parentURL, + conditions: DEFAULT_CONDITIONS, + }); - const resolveResponse = await this._resolve( - specifier, { parentURL, conditions: DEFAULT_CONDITIONS }, defaultResolve); if (typeof resolveResponse !== 'object') { throw new ERR_INVALID_RETURN_VALUE( 'object', 'loader resolve', resolveResponse); @@ -95,8 +109,11 @@ class Loader { } async getFormat(url) { - const getFormatResponse = await this._getFormat( - url, {}, defaultGetFormat); + const getFormatResponse = + await bottomLoaderRPC.getFormat({ + clientId: threadId, + url, + }); if (typeof getFormatResponse !== 'object') { throw new ERR_INVALID_RETURN_VALUE( 'object', 'loader getFormat', getFormatResponse); @@ -112,7 +129,7 @@ class Loader { return format; } - if (this._resolve !== defaultResolve) { + if (bottomLoaderRPC !== defaultLoaderWorker) { try { new URL(url); } catch { @@ -163,38 +180,6 @@ class Loader { return module.getNamespace(); } - hook(hooks) { - const { - resolve, - dynamicInstantiate, - getFormat, - getSource, - transformSource, - getGlobalPreloadCode, - } = hooks; - - // Use .bind() to avoid giving access to the Loader instance when called. - if (resolve !== undefined) - this._resolve = FunctionPrototypeBind(resolve, null); - if (dynamicInstantiate !== undefined) { - process.emitWarning( - 'The dynamicInstantiate loader hook has been removed.'); - } - if (getFormat !== undefined) { - this._getFormat = FunctionPrototypeBind(getFormat, null); - } - if (getSource !== undefined) { - this._getSource = FunctionPrototypeBind(getSource, null); - } - if (transformSource !== undefined) { - this._transformSource = FunctionPrototypeBind(transformSource, null); - } - if (getGlobalPreloadCode !== undefined) { - this._getGlobalPreloadCode = - FunctionPrototypeBind(getGlobalPreloadCode, null); - } - } - runGlobalPreloadCode() { if (!this._getGlobalPreloadCode) { return; @@ -222,6 +207,13 @@ class Loader { }); } + // Use this to avoid .then() exports being returned + async importWrapped(specifier, parent) { + const job = await this.getModuleJob(specifier, parent); + const { module } = await job.run(); + return { namespace: module.getNamespace() }; + } + async getModuleJob(specifier, parentURL) { const url = await this.resolve(specifier, parentURL); const format = await this.getFormat(url); @@ -249,3 +241,23 @@ class Loader { ObjectSetPrototypeOf(Loader.prototype, null); exports.Loader = Loader; + +let bottomLoaderRPC; +const { + RemoteLoaderWorker, +} = require('internal/modules/esm/ipc_types'); + +exports.getBottomLoaderRPC = () => { + assert(bottomLoaderRPC !== undefined); + return bottomLoaderRPC; +}; + +exports.initUserLoaders = (bottomLoader) => { + if (!bottomLoader) { + bottomLoaderRPC = defaultLoaderWorker; + return; + } + const { emitExperimentalWarning } = require('internal/util'); + emitExperimentalWarning('--experimental-loader'); + bottomLoaderRPC = new RemoteLoaderWorker(bottomLoader); +}; diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index f314ba96b3476c..234b4795cc768f 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -23,10 +23,6 @@ const { } = require('internal/modules/cjs/helpers'); const CJSModule = require('internal/modules/cjs/loader').Module; const internalURLModule = require('internal/url'); -const { defaultGetSource } = require( - 'internal/modules/esm/get_source'); -const { defaultTransformSource } = require( - 'internal/modules/esm/transform_source'); const createDynamicModule = require( 'internal/modules/esm/create_dynamic_module'); const { fileURLToPath, URL } = require('url'); @@ -111,11 +107,9 @@ function initializeImportMeta(meta, { url }) { // Strategy for loading a standard JavaScript module translators.set('module', async function moduleStrategy(url) { - let { source } = await this._getSource( - url, { format: 'module' }, defaultGetSource); + let { source } = await this._getSource({ url, format: 'module' }); assertBufferSource(source, true, 'getSource'); - ({ source } = await this._transformSource( - source, { url, format: 'module' }, defaultTransformSource)); + ({ source } = await this._transformSource({ source, url, format: 'module' })); source = stringify(source); maybeCacheSourceMap(url, source); debug(`Translating StandardModule ${url}`); @@ -189,11 +183,9 @@ translators.set('json', async function jsonStrategy(url) { }); } } - let { source } = await this._getSource( - url, { format: 'json' }, defaultGetSource); + let { source } = await this._getSource({ url, format: 'json' }); assertBufferSource(source, true, 'getSource'); - ({ source } = await this._transformSource( - source, { url, format: 'json' }, defaultTransformSource)); + ({ source } = await this._transformSource({ source, url, format: 'json' })); source = stringify(source); if (pathname) { // A require call could have been called on the same file during loading and @@ -233,11 +225,9 @@ translators.set('json', async function jsonStrategy(url) { // Strategy for loading a wasm module translators.set('wasm', async function(url) { emitExperimentalWarning('Importing Web Assembly modules'); - let { source } = await this._getSource( - url, { format: 'wasm' }, defaultGetSource); + let { source } = await this._getSource({ url, format: 'wasm' }); assertBufferSource(source, false, 'getSource'); - ({ source } = await this._transformSource( - source, { url, format: 'wasm' }, defaultTransformSource)); + ({ source } = await this._transformSource({ source, url, format: 'wasm' })); assertBufferSource(source, false, 'transformSource'); debug(`Translating WASMModule ${url}`); let compiled; diff --git a/lib/internal/modules/esm/worker.js b/lib/internal/modules/esm/worker.js new file mode 100644 index 00000000000000..275b925b2fe019 --- /dev/null +++ b/lib/internal/modules/esm/worker.js @@ -0,0 +1,180 @@ +/* eslint-disable */ +const { + connectIncoming, + RemoteLoaderWorker +} = require('internal/modules/esm/ipc_types'); +const publicWorker = require('worker_threads'); +const { + parentPort, + workerData, +} = publicWorker; +if (!parentPort) return; +parentPort.close(); +// Hide parentPort and workerData from loader hooks. +publicWorker.parentPort = null; +publicWorker.workerData = null; +const esmLoader = require('internal/process/esm_loader'); +const { defaultResolve } = require('internal/modules/esm/resolve'); +const { defaultGetFormat } = require('internal/modules/esm/get_format'); +const { defaultGetSource } = require('internal/modules/esm/get_source'); +const { defaultTransformSource } = require( + 'internal/modules/esm/transform_source'); + +const defaultLoaderWorker = { + resolveImportURL(params) { + return defaultResolve(params.specifier, { + parentURL: params.base, + conditions: params.conditions, + }); + }, + + getFormat({ url }) { + return defaultGetFormat(url); + }, + + getSource({ url, format }) { + return defaultGetSource(url, { format }); + }, + + transformSource({ source, url, format }) { + return defaultTransformSource(source, { url, format }); + }, +}; + +{ + const { + insideBelowPort, + insideAbovePort + } = workerData; + let parentLoaderAPI; + if (!insideAbovePort) { + parentLoaderAPI = defaultLoaderWorker; + } else { + const aboveDelegates = []; + let rpcIndex = 0; + const pendingAbove = []; + parentLoaderAPI = { + resolve(params) { + return new Promise((f, r) => { + if (rpcIndex >= aboveDelegates.length) { + pendingAbove.push({ + params, + return: f, + throw: r + }); + return; + } + const rpcAbove = aboveDelegates[rpcIndex]; + rpcIndex++; + sendAbove(params, rpcAbove, f, r); + }); + } + }; + function addAbove(port) { + if (workerData.nothingAbove === true) throw new Error(); + port.on('close', () => { + const i = aboveDelegates.indexOf(rpcAbove); + aboveDelegates.splice(i, 1); + if (i < rpcIndex) rpcIndex--; + }); + let rpcAbove = new RemoteLoaderWorker(insideAbovePort); + + aboveDelegates.push(rpcAbove); + const pending = [...pendingAbove]; + pendingAbove.length = 0; + for (const { params, return: f, throw: r } of pending) { + sendAbove(params, rpcAbove, f, r); + } + } + + async function sendAbove(params, rpcAbove, f, r) { + try { + const value = ResolveResponse.fromOrNull( + await rpcAbove.send(new ResolveRequest(params)) + ); + if (value !== null) return f(value); + else return r(new Error('unknown resolve response')); + } catch (e) { + return r(e); + } + } + + addAbove(insideAbovePort); + } + + const userModule = esmLoader.ESMLoader.importWrapped(workerData.loaderHREF).catch( + (err) => { + const { decorateErrorStack } = require('internal/util'); + decorateErrorStack(err); + internalBinding('errors').triggerUncaughtException( + err, + true /* fromPromise */ + ); + } + ); + + async function forwardResolveHook(specifier, options) { + return parentLoaderAPI.resolveImportURL({ + specifier, + base: options.parentURL, + conditions: options.conditions, + }); + } + + async function forwardGetFormatHook(url) { + return parentLoaderAPI.getFormat({ url }); + } + + async function forwardGetSourceHook(url, { format }) { + return parentLoaderAPI.getSource({ url, format }); + } + + async function forwardTransformSourceHook(source, { url, format }) { + return parentLoaderAPI.transformSource({ source, format, url }); + } + + const userLoaderWorker = { + async resolveImportURL(params) { + const { namespace: {resolve: resolveHandler}} = await userModule; + if (!resolveHandler) { + return parentLoaderAPI.resolveImportURL(params); + } + const options = { + parentURL: params.base, + conditions: params.conditions, + }; + return resolveHandler(params.specifier, options, forwardResolveHook); + }, + + async getFormat({ url }) { + const { namespace: {getFormat: getFormatHandler}} = await userModule; + if (!getFormatHandler) { + return parentLoaderAPI.getFormat({ url }); + } + return getFormatHandler(url, {}, forwardGetFormatHook); + }, + + async getSource({ url, format }) { + const { namespace: {getSource: getSourceHandler}} = await userModule; + if (!getSourceHandler) { + return parentLoaderAPI.getSource({ url, format }); + } + return getSourceHandler(url, { format }, forwardGetSourceHook); + }, + + async transformSource({ url, format, source }) { + const { namespace: {transformSource: transformSourceHandler}} = await userModule; + if (!transformSourceHandler) { + return parentLoaderAPI.transformSource({ url, format, source }); + } + return transformSourceHandler(source, { format, url }, forwardTransformSourceHook); + }, + + async addBelowPort({ port }) { + connectIncoming(port, this); + return null; + } + }; + + userLoaderWorker.addBelowPort({ port: insideBelowPort }); +} diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 5f8b1f53d33768..aade21988cd25d 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -37,16 +37,17 @@ function shouldUseESMLoader(mainPath) { return pkg && pkg.data.type === 'module'; } -function runMainESM(mainPath) { +async function runMainESM(mainPath) { const esmLoader = require('internal/process/esm_loader'); const { pathToFileURL } = require('internal/url'); const { hasUncaughtExceptionCaptureCallback } = require('internal/process/execution'); - return esmLoader.initializeLoader().then(() => { + + try { const main = path.isAbsolute(mainPath) ? pathToFileURL(mainPath).href : mainPath; - return esmLoader.ESMLoader.import(main); - }).catch((e) => { + await esmLoader.ESMLoader.import(main); + } catch (e) { if (hasUncaughtExceptionCaptureCallback()) { process._fatalException(e); return; @@ -55,7 +56,7 @@ function runMainESM(mainPath) { e, true /* fromPromise */ ); - }); + } } // For backwards compatibility, we have to run a bunch of diff --git a/lib/internal/process/esm_loader.js b/lib/internal/process/esm_loader.js index fe47098fde2f63..9e164584951d02 100644 --- a/lib/internal/process/esm_loader.js +++ b/lib/internal/process/esm_loader.js @@ -4,7 +4,6 @@ const { ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING, } = require('internal/errors').codes; const { Loader } = require('internal/modules/esm/loader'); -const { pathToFileURL } = require('internal/url'); const { getModuleFromWrap, } = require('internal/vm/module'); @@ -31,31 +30,5 @@ exports.importModuleDynamicallyCallback = async function(wrap, specifier) { throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING(); }; -let ESMLoader = new Loader(); +const ESMLoader = new Loader(); exports.ESMLoader = ESMLoader; - -exports.initializeLoader = initializeLoader; -async function initializeLoader() { - const { getOptionValue } = require('internal/options'); - const userLoader = getOptionValue('--experimental-loader'); - if (!userLoader) - return; - let cwd; - try { - cwd = process.cwd() + '/'; - } catch { - cwd = 'file:///'; - } - // If --experimental-loader is specified, create a loader with user hooks. - // Otherwise create the default loader. - const { emitExperimentalWarning } = require('internal/util'); - emitExperimentalWarning('--experimental-loader'); - return (async () => { - const hooks = - await ESMLoader.import(userLoader, pathToFileURL(cwd).href); - ESMLoader = new Loader(); - ESMLoader.hook(hooks); - ESMLoader.runGlobalPreloadCode(); - return exports.ESMLoader = ESMLoader; - })(); -} diff --git a/lib/internal/worker.js b/lib/internal/worker.js index 4f01f331349f4c..92046a546ebe32 100644 --- a/lib/internal/worker.js +++ b/lib/internal/worker.js @@ -82,8 +82,9 @@ if (isMainThread) { } class Worker extends EventEmitter { - constructor(filename, options = {}) { + constructor(filename, options = {}, internalMarker) { super(); + const internal = internalMarker === InternalWorker; debug(`[${threadId}] create new worker`, filename, options); if (options.execArgv && !ArrayIsArray(options.execArgv)) { throw new ERR_INVALID_ARG_TYPE('options.execArgv', @@ -99,7 +100,7 @@ class Worker extends EventEmitter { } let url; - if (options.eval) { + if (options.eval || internal) { if (typeof filename !== 'string') { throw new ERR_INVALID_ARG_VALUE( 'options.eval', @@ -121,7 +122,7 @@ class Worker extends EventEmitter { } else if (path.isAbsolute(filename) || /^\.\.?[\\/]/.test(filename)) { filename = path.resolve(filename); url = pathToFileURL(filename); - } else { + } else if (!internal) { throw new ERR_WORKER_PATH(filename); } @@ -192,13 +193,30 @@ class Worker extends EventEmitter { this[kPublicPort] = port1; this[kPublicPort].on('message', (message) => this.emit('message', message)); setupPortReferencing(this[kPublicPort], this, 'message'); + let outsideBelowPort, insideBelowPort; + if (!internal) { + const loaderRPC = + require('internal/modules/esm/loader').getBottomLoaderRPC(); + if (loaderRPC && loaderRPC.addBelowPort) { + ({ + port1: outsideBelowPort, + port2: insideBelowPort + } = new MessageChannel()); + loaderRPC.addBelowPort({ + port: insideBelowPort + }, [insideBelowPort]); + transferList.push(outsideBelowPort); + } + } this[kPort].postMessage({ argv, type: messageTypes.LOAD_SCRIPT, filename, doEval: !!options.eval, + internal, cwdCounter: cwdCounter || workerIo.sharedCwdCounter, workerData: options.workerData, + loaderPort: outsideBelowPort, publicPort: port2, manifestSrc: getOptionValue('--experimental-policy') ? require('internal/process/policy').src : @@ -359,6 +377,10 @@ class Worker extends EventEmitter { } } +function InternalWorker(a, b) { + return new Worker(a, b, InternalWorker); +} + function pipeWithoutWarning(source, dest) { const sourceMaxListeners = source._maxListeners; const destMaxListeners = dest._maxListeners; @@ -405,4 +427,5 @@ module.exports = { !isMainThread ? makeResourceLimits(resourceLimitsRaw) : {}, threadId, Worker, + InternalWorker, }; diff --git a/node.gyp b/node.gyp index 63a5d341d8bb02..847300f3735db3 100644 --- a/node.gyp +++ b/node.gyp @@ -166,11 +166,13 @@ 'lib/internal/modules/esm/create_dynamic_module.js', 'lib/internal/modules/esm/get_format.js', 'lib/internal/modules/esm/get_source.js', + 'lib/internal/modules/esm/ipc_types.js', 'lib/internal/modules/esm/module_job.js', 'lib/internal/modules/esm/module_map.js', 'lib/internal/modules/esm/resolve.js', 'lib/internal/modules/esm/transform_source.js', 'lib/internal/modules/esm/translators.js', + 'lib/internal/modules/esm/worker.js', 'lib/internal/net.js', 'lib/internal/options.js', 'lib/internal/policy/manifest.js', diff --git a/test/es-module/test-esm-loader-isolation.mjs b/test/es-module/test-esm-loader-isolation.mjs new file mode 100644 index 00000000000000..ce5cf078203dcb --- /dev/null +++ b/test/es-module/test-esm-loader-isolation.mjs @@ -0,0 +1,37 @@ +// Flags: --experimental-loader ./test/fixtures/es-module-loaders/isolation-hook.mjs +import { mustCall } from '../common/index.mjs'; +import assert from 'assert'; +import { randomBytes } from 'crypto'; +import { fileURLToPath } from 'url'; +import { Worker, isMainThread, parentPort } from 'worker_threads'; + +import { + parentPort as hookParentPort, + workerData as hookWorkerData, +} from 'test!worker_threads'; +import { globalValue } from 'test!globalValue'; +import { + identity as hookIdentity, + threadId as hookThreadId, +} from 'test!identity'; + +assert.strictEqual(hookParentPort, null); +assert.strictEqual(hookWorkerData, null); + +assert.strictEqual(globalValue, 42); +assert.notStrictEqual(globalThis.globalValue, globalValue); + +const hookIdentityData = { + static: { hookIdentity, hookThreadId }, + unique: randomBytes(16).toString('hex'), +}; + +if (isMainThread) { + const worker = new Worker(fileURLToPath(import.meta.url)); + worker.once('message', mustCall((workerMessage) => { + assert.deepStrictEqual(workerMessage.static, hookIdentityData.static); + assert.notDeepStrictEqual(workerMessage.unique, hookIdentityData.unique); + })); +} else { + parentPort.postMessage(hookIdentityData); +} diff --git a/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs index f476c676cdea5b..3b22c343025927 100644 --- a/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs +++ b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs @@ -13,8 +13,8 @@ export function getGlobalPreloadCode() { `; } -export function resolve(specifier, context, defaultResolve) { - const def = defaultResolve(specifier, context); +export async function resolve(specifier, context, defaultResolve) { + const def = await defaultResolve(specifier, context); if (def.url.startsWith('nodejs:')) { return { url: `custom-${def.url}`, diff --git a/test/fixtures/es-module-loaders/isolation-hook.mjs b/test/fixtures/es-module-loaders/isolation-hook.mjs new file mode 100644 index 00000000000000..0cf56cb112f85e --- /dev/null +++ b/test/fixtures/es-module-loaders/isolation-hook.mjs @@ -0,0 +1,38 @@ +import { randomBytes } from 'crypto'; +import { parentPort, workerData, threadId } from 'worker_threads'; + +globalThis.globalValue = 42; + +// Create a random value unique to this instance of the hooks. Useful to ensure +// that threads share the same loader. +const identity = randomBytes(16).toString('hex'); + +function valueResolution(value) { + const src = Object.entries(value).map( + ([exportName, exportValue]) => + `export const ${exportName} = ${JSON.stringify(exportValue)};` + ).join('\n'); + return { + url: `data:text/javascript;base64,${Buffer.from(src).toString('base64')}`, + format: 'module', + }; +} + +export async function resolve(specifier, referrer, parentResolve) { + if (!specifier.startsWith('test!')) { + return parentResolve(specifier, referrer); + } + + switch (specifier.substr(5)) { + case 'worker_threads': + return valueResolution({ parentPort, workerData }); + + case 'globalValue': + return valueResolution({ globalValue: globalThis.globalValue }); + + case 'identity': + return valueResolution({ threadId, identity }) + } + + throw new Error('Invalid test case'); +} diff --git a/test/fixtures/es-module-loaders/loader-with-dep.mjs b/test/fixtures/es-module-loaders/loader-with-dep.mjs index da7d44ae793e22..5d9c909dd46997 100644 --- a/test/fixtures/es-module-loaders/loader-with-dep.mjs +++ b/test/fixtures/es-module-loaders/loader-with-dep.mjs @@ -3,9 +3,9 @@ import {createRequire} from '../../common/index.mjs'; const require = createRequire(import.meta.url); const dep = require('./loader-dep.js'); -export function resolve (specifier, { parentURL }, defaultResolve) { +export async function resolve (specifier, { parentURL }, defaultResolve) { return { - url: defaultResolve(specifier, {parentURL}, defaultResolve).url, + url: (await defaultResolve(specifier, {parentURL}, defaultResolve)).url, format: dep.format }; } diff --git a/test/message/esm_display_syntax_error_import.out b/test/message/esm_display_syntax_error_import.out index 387a63a734b512..6f1d3e5e11c083 100644 --- a/test/message/esm_display_syntax_error_import.out +++ b/test/message/esm_display_syntax_error_import.out @@ -5,3 +5,4 @@ SyntaxError: The requested module '../fixtures/es-module-loaders/module-named-ex at ModuleJob._instantiate (internal/modules/esm/module_job.js:*:*) at async ModuleJob.run (internal/modules/esm/module_job.js:*:*) at async Loader.import (internal/modules/esm/loader.js:*:*) + at async runMainESM (internal/modules/run_main.js:*:*) diff --git a/test/message/esm_display_syntax_error_import_module.out b/test/message/esm_display_syntax_error_import_module.out index ae8b99d55fef20..7642282f8f76a1 100644 --- a/test/message/esm_display_syntax_error_import_module.out +++ b/test/message/esm_display_syntax_error_import_module.out @@ -5,3 +5,4 @@ SyntaxError: The requested module './module-named-exports.mjs' does not provide at ModuleJob._instantiate (internal/modules/esm/module_job.js:*:*) at async ModuleJob.run (internal/modules/esm/module_job.js:*:*) at async Loader.import (internal/modules/esm/loader.js:*:*) + at async runMainESM (internal/modules/run_main.js:*:*) diff --git a/test/message/esm_loader_not_found.out b/test/message/esm_loader_not_found.out index 1d2aa957150082..06375e9896f1ac 100644 --- a/test/message/esm_loader_not_found.out +++ b/test/message/esm_loader_not_found.out @@ -1,18 +1,26 @@ (node:*) ExperimentalWarning: --experimental-loader is an experimental feature. This feature could change at any time (Use `node --trace-warnings ...` to show where the warning was created) -internal/modules/run_main.js:* - internalBinding('errors').triggerUncaughtException( - ^ + +events.js:* + throw er; // Unhandled 'error' event + ^ Error [ERR_MODULE_NOT_FOUND]: Cannot find package 'i-dont-exist' imported from * at packageResolve (internal/modules/esm/resolve.js:*:*) at moduleResolve (internal/modules/esm/resolve.js:*:*) - at Loader.defaultResolve [as _resolve] (internal/modules/esm/resolve.js:*:*) + at defaultResolve (internal/modules/esm/resolve.js:*:*) + at Object.resolveImportURL (internal/modules/esm/loader.js:*:*) at Loader.resolve (internal/modules/esm/loader.js:*:*) at Loader.getModuleJob (internal/modules/esm/loader.js:*:*) - at Loader.import (internal/modules/esm/loader.js:*:*) - at internal/process/esm_loader.js:*:* - at Object.initializeLoader (internal/process/esm_loader.js:*:*) - at runMainESM (internal/modules/run_main.js:*:*) - at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js:*:*) { + at Loader.importWrapped (internal/modules/esm/loader.js:*:*) + at internal/modules/esm/worker.js:*:* + at NativeModule.compileForInternalLoader (internal/bootstrap/loaders.js:*:*) + at nativeModuleRequire (internal/bootstrap/loaders.js:*:*) +Emitted 'error' event on Worker instance at: + at Worker.[kOnErrorMessage] (internal/worker.js:*:*) + at Worker.[kOnMessage] (internal/worker.js:*:*) + at MessagePort. (internal/worker.js:*:*) + at MessagePort.emit (events.js:*:*) + at MessagePort.onmessage (internal/worker/io.js:*:*) + at MessagePort.exports.emitMessage (internal/per_context/messageport.js:*:*) { code: 'ERR_MODULE_NOT_FOUND' } diff --git a/test/message/esm_loader_not_found_cjs_hint_bare.out b/test/message/esm_loader_not_found_cjs_hint_bare.out index e56f1da0f6e76e..eb769baca2ea98 100644 --- a/test/message/esm_loader_not_found_cjs_hint_bare.out +++ b/test/message/esm_loader_not_found_cjs_hint_bare.out @@ -7,7 +7,8 @@ Did you mean to import some_module/obj.js? at finalizeResolution (internal/modules/esm/resolve.js:*:*) at packageResolve (internal/modules/esm/resolve.js:*:*) at moduleResolve (internal/modules/esm/resolve.js:*:*) - at Loader.defaultResolve [as _resolve] (internal/modules/esm/resolve.js:*:*) + at defaultResolve (internal/modules/esm/resolve.js:*:*) + at Object.resolveImportURL (internal/modules/esm/loader.js:*:*) at Loader.resolve (internal/modules/esm/loader.js:*:*) at Loader.getModuleJob (internal/modules/esm/loader.js:*:*) at ModuleWrap. (internal/modules/esm/module_job.js:*:*) diff --git a/test/message/esm_loader_not_found_cjs_hint_relative.out b/test/message/esm_loader_not_found_cjs_hint_relative.out index 76df3163bb728c..124f8617b9c406 100644 --- a/test/message/esm_loader_not_found_cjs_hint_relative.out +++ b/test/message/esm_loader_not_found_cjs_hint_relative.out @@ -1,20 +1,27 @@ (node:*) ExperimentalWarning: --experimental-loader is an experimental feature. This feature could change at any time (Use `node --trace-warnings ...` to show where the warning was created) -internal/modules/run_main.js:* - internalBinding('errors').triggerUncaughtException( - ^ +events.js:* + throw er; // Unhandled 'error' event + ^ Error [ERR_MODULE_NOT_FOUND]: Cannot find module '*test*common*fixtures' imported from * Did you mean to import ./test/common/fixtures.js? at finalizeResolution (internal/modules/esm/resolve.js:*:*) at moduleResolve (internal/modules/esm/resolve.js:*:*) - at Loader.defaultResolve [as _resolve] (internal/modules/esm/resolve.js:*:*) + at defaultResolve (internal/modules/esm/resolve.js:*:*) + at Object.resolveImportURL (internal/modules/esm/loader.js:*:*) at Loader.resolve (internal/modules/esm/loader.js:*:*) at Loader.getModuleJob (internal/modules/esm/loader.js:*:*) - at Loader.import (internal/modules/esm/loader.js:*:*) - at internal/process/esm_loader.js:*:* - at Object.initializeLoader (internal/process/esm_loader.js:*:*) - at runMainESM (internal/modules/run_main.js:*:*) - at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js:*:*) { + at Loader.importWrapped (internal/modules/esm/loader.js:*:*) + at internal/modules/esm/worker.js:*:* + at NativeModule.compileForInternalLoader (internal/bootstrap/loaders.js:*:*) + at nativeModuleRequire (internal/bootstrap/loaders.js:*:*) +Emitted 'error' event on Worker instance at: + at Worker.[kOnErrorMessage] (internal/worker.js:*:*) + at Worker.[kOnMessage] (internal/worker.js:*:*) + at MessagePort. (internal/worker.js:*:*) + at MessagePort.emit (events.js:*:*) + at MessagePort.onmessage (internal/worker/io.js:*:*) + at MessagePort.exports.emitMessage (internal/per_context/messageport.js:*:*) { code: 'ERR_MODULE_NOT_FOUND' } diff --git a/test/message/esm_loader_syntax_error.out b/test/message/esm_loader_syntax_error.out index 3aee72d423b1a1..26195ec45f967c 100644 --- a/test/message/esm_loader_syntax_error.out +++ b/test/message/esm_loader_syntax_error.out @@ -1,9 +1,19 @@ (node:*) ExperimentalWarning: --experimental-loader is an experimental feature. This feature could change at any time (Use `node --trace-warnings ...` to show where the warning was created) -file://*/test/fixtures/es-module-loaders/syntax-error.mjs:2 + +events.js:* + throw er; // Unhandled 'error' event + ^ +file:///*/test/fixtures/es-module-loaders/syntax-error.mjs:2 await async () => 0; ^^^^^ - SyntaxError: Unexpected reserved word at Loader.moduleStrategy (internal/modules/esm/translators.js:*:*) at async link (internal/modules/esm/module_job.js:*:*) +Emitted 'error' event on Worker instance at: + at Worker.[kOnErrorMessage] (internal/worker.js:*:*) + at Worker.[kOnMessage] (internal/worker.js:*:*) + at MessagePort. (internal/worker.js:*:*) + at MessagePort.emit (events.js:*:*) + at MessagePort.onmessage (internal/worker/io.js:*:*) + at MessagePort.exports.emitMessage (internal/per_context/messageport.js:*:*) diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 62775b9527998b..3be84bb46ad5af 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -30,6 +30,7 @@ const expectedModules = new Set([ 'Internal Binding types', 'Internal Binding url', 'Internal Binding util', + 'Internal Binding worker', 'NativeModule buffer', 'NativeModule events', 'NativeModule fs', @@ -54,6 +55,7 @@ const expectedModules = new Set([ 'NativeModule internal/modules/esm/create_dynamic_module', 'NativeModule internal/modules/esm/get_format', 'NativeModule internal/modules/esm/get_source', + 'NativeModule internal/modules/esm/ipc_types', 'NativeModule internal/modules/esm/loader', 'NativeModule internal/modules/esm/module_job', 'NativeModule internal/modules/esm/module_map',