From bbfe4a3b7ad3d3ce93f41aa2a232968f914223c8 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 2 Jan 2019 13:54:51 +0100 Subject: [PATCH 01/16] benchmark: add new module loading benchmarks --- benchmark/module/load-native.js | 19 ++++++++++ benchmark/module/module-loader-deep.js | 51 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 benchmark/module/load-native.js create mode 100644 benchmark/module/module-loader-deep.js diff --git a/benchmark/module/load-native.js b/benchmark/module/load-native.js new file mode 100644 index 00000000000000..a933bf38932e9b --- /dev/null +++ b/benchmark/module/load-native.js @@ -0,0 +1,19 @@ +'use strict'; +const common = require('../common.js'); + +const bench = common.createBenchmark(main, { + path: ['vm', 'util'], + n: [1000], + useCache: ['true', 'false'] +}); + +function main({ n, path, useCache }) { + if (useCache) + require(path); + + bench.start(); + for (var i = 0; i < n; i++) { + require(path); + } + bench.end(n); +} diff --git a/benchmark/module/module-loader-deep.js b/benchmark/module/module-loader-deep.js new file mode 100644 index 00000000000000..f686b8df47dfba --- /dev/null +++ b/benchmark/module/module-loader-deep.js @@ -0,0 +1,51 @@ +'use strict'; +const fs = require('fs'); +const path = require('path'); +const common = require('../common.js'); + +const tmpdir = require('../../test/common/tmpdir'); +const benchmarkDirectory = path.join(tmpdir.path, 'nodejs-benchmark-module'); + +const bench = common.createBenchmark(main, { + ext: ['', '.js'], + files: [1e3], + cache: ['true', 'false'] +}); + +function main({ ext, cache, files }) { + tmpdir.refresh(); + fs.mkdirSync(benchmarkDirectory); + fs.writeFileSync( + `${benchmarkDirectory}/a.js`, + 'module.exports = {};' + ); + for (var i = 0; i <= files; i++) { + fs.mkdirSync(`${benchmarkDirectory}/${i}`); + fs.writeFileSync( + `${benchmarkDirectory}/${i}/package.json`, + '{"main": "index.js"}' + ); + fs.writeFileSync( + `${benchmarkDirectory}/${i}/index.js`, + `require('../a${ext}');` + ); + } + + measureDir(cache === 'true', files); + + tmpdir.refresh(); +} + +function measureDir(cache, files) { + var i; + if (cache) { + for (i = 0; i <= files; i++) { + require(`${benchmarkDirectory}/${i}`); + } + } + bench.start(); + for (i = 0; i <= files; i++) { + require(`${benchmarkDirectory}/${i}`); + } + bench.end(files); +} From 9c54e43ef7e40a48555cb08c223e3ecd72d97fb0 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Sun, 30 Dec 2018 18:24:28 +0100 Subject: [PATCH 02/16] modules: significantly improve repeated requires 1) It adds more benchmark options to properly verify the gains. This makes sure the benchmark also tests requiring the same module again instead of only loading each module only once. 2) Remove dead code: The array check is obsolete as this function will only be called internally with preprepared data which is always an array. 3) Simpler code It was possible to use a more direct logic to prevent some branches. 4) Inline try catch The function is not required anymore, since V8 is able to produce performant code with it. 5) var -> let / const & less lines 6) Update require.extensions description The comment was outdated. 7) Improve extension handling This is a performance optimization to prevent loading the extensions on each uncached require call. It uses proxies to intercept changes and receives the necessary informations by doing that. --- benchmark/module/module-loader.js | 53 +- doc/api/deprecations.md | 10 +- doc/api/modules.md | 57 +- lib/internal/bootstrap/loaders.js | 54 +- lib/internal/modules/cjs/helpers.js | 81 +- lib/internal/modules/cjs/loader.js | 749 +++++++++--------- lib/repl.js | 2 +- test/es-module/test-esm-namespace.mjs | 2 +- test/fixtures/module-require-depth/one.js | 11 - test/fixtures/module-require-depth/two.js | 11 - test/fixtures/require-resolve.js | 24 +- test/message/core_line_numbers.out | 6 +- test/message/error_exit.out | 7 +- .../events_unhandled_error_common_trace.out | 8 +- .../events_unhandled_error_nexttick.out | 12 +- .../events_unhandled_error_sameline.out | 9 +- test/message/if-error-has-good-stack.out | 6 +- test/message/nexttick_throw.out | 2 +- .../undefined_reference_in_new_context.out | 2 +- test/message/vm_display_runtime_error.out | 12 +- test/message/vm_display_syntax_error.out | 12 +- .../message/vm_dont_display_runtime_error.out | 6 +- test/message/vm_dont_display_syntax_error.out | 6 +- test/parallel/test-cwd-enoent-preload.js | 1 - test/parallel/test-module-relative-lookup.js | 5 +- test/parallel/test-module-require-depth.js | 16 - test/parallel/test-repl.js | 1 + test/parallel/test-require-dot.js | 11 +- 28 files changed, 555 insertions(+), 621 deletions(-) delete mode 100644 test/fixtures/module-require-depth/one.js delete mode 100644 test/fixtures/module-require-depth/two.js delete mode 100644 test/parallel/test-module-require-depth.js diff --git a/benchmark/module/module-loader.js b/benchmark/module/module-loader.js index e780d6376b5e8d..84aa62505c5b8d 100644 --- a/benchmark/module/module-loader.js +++ b/benchmark/module/module-loader.js @@ -4,18 +4,20 @@ const path = require('path'); const common = require('../common.js'); const tmpdir = require('../../test/common/tmpdir'); -const benchmarkDirectory = path.join(tmpdir.path, 'nodejs-benchmark-module'); +let benchmarkDirectory = path.join(tmpdir.path, 'nodejs-benchmark-module'); const bench = common.createBenchmark(main, { - n: [5e4], - fullPath: ['true', 'false'], - useCache: ['true', 'false'] + name: ['', '/', '/index.js'], + dir: ['rel', 'abs'], + files: [5e2], + n: [1, 1e3], + cache: ['true', 'false'] }); -function main({ n, fullPath, useCache }) { +function main({ n, name, cache, files, dir }) { tmpdir.refresh(); - try { fs.mkdirSync(benchmarkDirectory); } catch {} - for (var i = 0; i <= n; i++) { + fs.mkdirSync(benchmarkDirectory); + for (var i = 0; i <= files; i++) { fs.mkdirSync(`${benchmarkDirectory}${i}`); fs.writeFileSync( `${benchmarkDirectory}${i}/package.json`, @@ -27,38 +29,25 @@ function main({ n, fullPath, useCache }) { ); } - if (fullPath === 'true') - measureFull(n, useCache === 'true'); - else - measureDir(n, useCache === 'true'); + if (dir === 'rel') + benchmarkDirectory = path.relative(__dirname, benchmarkDirectory); - tmpdir.refresh(); -} + measureDir(n, cache === 'true', files, name); -function measureFull(n, useCache) { - var i; - if (useCache) { - for (i = 0; i <= n; i++) { - require(`${benchmarkDirectory}${i}/index.js`); - } - } - bench.start(); - for (i = 0; i <= n; i++) { - require(`${benchmarkDirectory}${i}/index.js`); - } - bench.end(n); + tmpdir.refresh(); } -function measureDir(n, useCache) { +function measureDir(n, cache, files, name) { var i; - if (useCache) { - for (i = 0; i <= n; i++) { - require(`${benchmarkDirectory}${i}`); + if (cache) { + for (i = 0; i <= files; i++) { + require(`${benchmarkDirectory}${i}${name}`); } } bench.start(); - for (i = 0; i <= n; i++) { - require(`${benchmarkDirectory}${i}`); + for (i = 0; i <= files; i++) { + for (var j = 0; j < n; j++) + require(`${benchmarkDirectory}${i}${name}`); } - bench.end(n); + bench.end(n * files); } diff --git a/doc/api/deprecations.md b/doc/api/deprecations.md index c4fbacc0e0ae46..61a0c11699aa19 100644 --- a/doc/api/deprecations.md +++ b/doc/api/deprecations.md @@ -442,6 +442,9 @@ code. ### DEP0019: require('.') resolved outside directory -Type: Runtime +Type: End-of-Life -In certain cases, `require('.')` may resolve outside the package directory. -This behavior is deprecated and will be removed in a future major Node.js -release. +In certain cases, `require('.')` could resolve outside the package directory. +This behavior has been removed. ### DEP0020: Server.connections diff --git a/doc/api/modules.md b/doc/api/modules.md index 5292d2389760bd..ea9b3bed09a994 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -196,28 +196,26 @@ NODE_MODULES_PATHS(START) -Modules are cached after the first time they are loaded. This means -(among other things) that every call to `require('foo')` will get -exactly the same object returned, if it would resolve to the same file. +Modules are cached after the first time they are loaded. This means (among other +things) that every call to `require('foo')` will get exactly the same object +returned, if it would resolve to the same file. -Provided `require.cache` is not modified, multiple calls to -`require('foo')` will not cause the module code to be executed multiple times. -This is an important feature. With it, "partially done" objects can be returned, -thus allowing transitive dependencies to be loaded even when they would cause -cycles. +Provided `require.cache` is not modified, multiple calls to `require('foo')` +will not cause the module code to be executed multiple times. This is an +important feature. With it, "partially done" objects can be returned, thus +allowing transitive dependencies to be loaded even when they would cause cycles. -To have a module execute code multiple times, export a function, and call -that function. +To have a module execute code multiple times, export a function, and call that +function. ### Module Caching Caveats -Modules are cached based on their resolved filename. Since modules may -resolve to a different filename based on the location of the calling -module (loading from `node_modules` folders), it is not a *guarantee* -that `require('foo')` will always return the exact same object, if it -would resolve to different files. +Modules are cached based on their resolved filename. Since modules may resolve +to a different filename based on the location of the calling module (loading +from `node_modules` folders), it is not a *guarantee* that `require('foo')` will +always return the exact same object, if it would resolve to different files. Additionally, on case-insensitive file systems or operating systems, different resolved filenames can point to the same file, but the cache will still treat @@ -412,7 +410,7 @@ are not found elsewhere. On Windows, `NODE_PATH` is delimited by semicolons (`;`) instead of colons. `NODE_PATH` was originally created to support loading modules from -varying paths before the current [module resolution][] algorithm was frozen. +varying paths before the current [module resolution][] algorithm was defined. `NODE_PATH` is still supported, but is less necessary now that the Node.js ecosystem has settled on a convention for locating dependent modules. @@ -582,6 +580,10 @@ value from this object, the next `require` will reload the module. Note that this does not apply to [native addons][], for which reloading will result in an error. +Adding ore replacing entries is also possible. This cache is checked before +native modules and if such a name is added to the cache, no require call is +going to receive the native module anymore. Use with care! + #### require.extensions