From ce52a4be24bf9ec858c0996702948dd2b96c50c3 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Thu, 24 Mar 2022 21:37:08 +0800 Subject: [PATCH] build: add --node-snapshot-main configure option This adds a --build-snapshot runtime option which is currently only supported by the node_mksnapshot binary, and a --node-snapshot-main configure option that makes use it to run a custom script when building the embedded snapshot. The idea is to have this experimental feature in core as a configure-time feature for now, and investigate the renaming V8 bugs before we make it available to more users via making it a runtime option. PR-URL: https://github.com/nodejs/node/pull/42466 Reviewed-By: Darshan Sen Reviewed-By: Mohammed Keyvanzadeh Reviewed-By: Khaidi Chu Reviewed-By: Chengzhong Wu --- configure.py | 19 ++++ lib/internal/bootstrap/pre_execution.js | 12 +- lib/internal/main/mksnapshot.js | 142 ++++++++++++++++++++++++ node.gyp | 53 ++++++--- src/node.cc | 10 ++ src/node_binding.cc | 1 + src/node_external_reference.h | 1 + src/node_options.cc | 5 + src/node_options.h | 1 + src/node_snapshotable.cc | 81 +++++++++++++- src/node_snapshotable.h | 2 + tools/snapshot/node_mksnapshot.cc | 74 +++++++++--- 12 files changed, 365 insertions(+), 36 deletions(-) create mode 100644 lib/internal/main/mksnapshot.js diff --git a/configure.py b/configure.py index fed2688c792..57e94e61c6d 100755 --- a/configure.py +++ b/configure.py @@ -798,6 +798,13 @@ default=False, help='node will load builtin modules from disk instead of from binary') +parser.add_argument('--node-snapshot-main', + action='store', + dest='node_snapshot_main', + default=None, + help='Run a file when building the embedded snapshot. Currently ' + + 'experimental.') + # Create compile_commands.json in out/Debug and out/Release. parser.add_argument('-C', action='store_true', @@ -1225,6 +1232,18 @@ def configure_node(o): o['variables']['want_separate_host_toolset'] = int(cross_compiling) + if options.node_snapshot_main is not None: + if options.shared: + # This should be possible to fix, but we will need to refactor the + # libnode target to avoid building it twice. + error('--node-snapshot-main is incompatible with --shared') + if options.without_node_snapshot: + error('--node-snapshot-main is incompatible with ' + + '--without-node-snapshot') + if cross_compiling: + error('--node-snapshot-main is incompatible with cross compilation') + o['variables']['node_snapshot_main'] = options.node_snapshot_main + if options.without_node_snapshot or options.node_builtin_modules_path: o['variables']['node_use_node_snapshot'] = 'false' else: diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index c306de7db63..7891370fbed 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -26,7 +26,8 @@ const { Buffer } = require('buffer'); const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes; const assert = require('internal/assert'); -function prepareMainThreadExecution(expandArgv1 = false) { +function prepareMainThreadExecution(expandArgv1 = false, + initialzeModules = true) { refreshRuntimeOptions(); // TODO(joyeecheung): this is also necessary for workers when they deserialize @@ -80,9 +81,13 @@ function prepareMainThreadExecution(expandArgv1 = false) { initializeSourceMapsHandlers(); initializeDeprecations(); initializeWASI(); + + if (!initialzeModules) { + return; + } + initializeCJSLoader(); initializeESMLoader(); - const CJSLoader = require('internal/modules/cjs/loader'); assert(!CJSLoader.hasLoadedAnyUserCJSModule); loadPreloadModules(); @@ -101,7 +106,8 @@ function patchProcessObject(expandArgv1) { ObjectDefineProperty(process, 'argv0', { enumerable: true, - configurable: false, + // Only set it to true during snapshot building. + configurable: getOptionValue('--build-snapshot'), value: process.argv[0] }); process.argv[0] = process.execPath; diff --git a/lib/internal/main/mksnapshot.js b/lib/internal/main/mksnapshot.js new file mode 100644 index 00000000000..3e1515a2d2e --- /dev/null +++ b/lib/internal/main/mksnapshot.js @@ -0,0 +1,142 @@ +'use strict'; + +const { + Error, + SafeSet, + SafeArrayIterator +} = primordials; + +const binding = internalBinding('mksnapshot'); +const { NativeModule } = require('internal/bootstrap/loaders'); +const { + compileSnapshotMain, +} = binding; + +const { + getOptionValue +} = require('internal/options'); + +const { + readFileSync +} = require('fs'); + +const supportedModules = new SafeSet(new SafeArrayIterator([ + // '_http_agent', + // '_http_client', + // '_http_common', + // '_http_incoming', + // '_http_outgoing', + // '_http_server', + '_stream_duplex', + '_stream_passthrough', + '_stream_readable', + '_stream_transform', + '_stream_wrap', + '_stream_writable', + // '_tls_common', + // '_tls_wrap', + 'assert', + 'assert/strict', + // 'async_hooks', + 'buffer', + // 'child_process', + // 'cluster', + 'console', + 'constants', + 'crypto', + // 'dgram', + // 'diagnostics_channel', + // 'dns', + // 'dns/promises', + // 'domain', + 'events', + 'fs', + 'fs/promises', + // 'http', + // 'http2', + // 'https', + // 'inspector', + // 'module', + // 'net', + 'os', + 'path', + 'path/posix', + 'path/win32', + // 'perf_hooks', + 'process', + 'punycode', + 'querystring', + // 'readline', + // 'repl', + 'stream', + 'stream/promises', + 'string_decoder', + 'sys', + 'timers', + 'timers/promises', + // 'tls', + // 'trace_events', + // 'tty', + 'url', + 'util', + 'util/types', + 'v8', + // 'vm', + // 'worker_threads', + // 'zlib', +])); + +const warnedModules = new SafeSet(); +function supportedInUserSnapshot(id) { + return supportedModules.has(id); +} + +function requireForUserSnapshot(id) { + if (!NativeModule.canBeRequiredByUsers(id)) { + // eslint-disable-next-line no-restricted-syntax + const err = new Error( + `Cannot find module '${id}'. ` + ); + err.code = 'MODULE_NOT_FOUND'; + throw err; + } + if (!supportedInUserSnapshot(id)) { + if (!warnedModules.has(id)) { + process.emitWarning( + `built-in module ${id} is not yet supported in user snapshots`); + warnedModules.add(id); + } + } + + return require(id); +} + +function main() { + const { + prepareMainThreadExecution + } = require('internal/bootstrap/pre_execution'); + + prepareMainThreadExecution(true, false); + process.once('beforeExit', function runCleanups() { + for (const cleanup of binding.cleanups) { + cleanup(); + } + }); + + const file = process.argv[1]; + const path = require('path'); + const filename = path.resolve(file); + const dirname = path.dirname(filename); + const source = readFileSync(file, 'utf-8'); + const snapshotMainFunction = compileSnapshotMain(filename, source); + + if (getOptionValue('--inspect-brk')) { + internalBinding('inspector').callAndPauseOnStart( + snapshotMainFunction, undefined, + requireForUserSnapshot, filename, dirname); + } else { + snapshotMainFunction(requireForUserSnapshot, filename, dirname); + } +} + +main(); diff --git a/node.gyp b/node.gyp index 9851e009205..6a1196c5093 100644 --- a/node.gyp +++ b/node.gyp @@ -7,6 +7,7 @@ 'node_use_dtrace%': 'false', 'node_use_etw%': 'false', 'node_no_browser_globals%': 'false', + 'node_snapshot_main%': '', 'node_use_node_snapshot%': 'false', 'node_use_v8_platform%': 'true', 'node_use_bundled_v8%': 'true', @@ -315,23 +316,47 @@ 'dependencies': [ 'node_mksnapshot', ], - 'actions': [ - { - 'action_name': 'node_mksnapshot', - 'process_outputs_as_sources': 1, - 'inputs': [ - '<(node_mksnapshot_exec)', - ], - 'outputs': [ - '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + 'conditions': [ + ['node_snapshot_main!=""', { + 'actions': [ + { + 'action_name': 'node_mksnapshot', + 'process_outputs_as_sources': 1, + 'inputs': [ + '<(node_mksnapshot_exec)', + '<(node_snapshot_main)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + ], + 'action': [ + '<(node_mksnapshot_exec)', + '--build-snapshot', + '<(node_snapshot_main)', + '<@(_outputs)', + ], + }, ], - 'action': [ - '<@(_inputs)', - '<@(_outputs)', + }, { + 'actions': [ + { + 'action_name': 'node_mksnapshot', + 'process_outputs_as_sources': 1, + 'inputs': [ + '<(node_mksnapshot_exec)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/node_snapshot.cc', + ], + 'action': [ + '<@(_inputs)', + '<@(_outputs)', + ], + }, ], - }, + }], ], - }, { + }, { 'sources': [ 'src/node_snapshot_stub.cc' ], diff --git a/src/node.cc b/src/node.cc index 95737c62bdb..1ecd220940c 100644 --- a/src/node.cc +++ b/src/node.cc @@ -492,6 +492,10 @@ MaybeLocal StartExecution(Environment* env, StartExecutionCallback cb) { return StartExecution(env, "internal/main/inspect"); } + if (per_process::cli_options->build_snapshot) { + return StartExecution(env, "internal/main/mksnapshot"); + } + if (per_process::cli_options->print_help) { return StartExecution(env, "internal/main/print_help"); } @@ -1184,6 +1188,12 @@ int Start(int argc, char** argv) { return result.exit_code; } + if (per_process::cli_options->build_snapshot) { + fprintf(stderr, + "--build-snapshot is not yet supported in the node binary\n"); + return 1; + } + { bool use_node_snapshot = per_process::cli_options->per_isolate->node_snapshot; diff --git a/src/node_binding.cc b/src/node_binding.cc index 6da455c786f..2302b3df836 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -60,6 +60,7 @@ V(js_udp_wrap) \ V(messaging) \ V(module_wrap) \ + V(mksnapshot) \ V(native_module) \ V(options) \ V(os) \ diff --git a/src/node_external_reference.h b/src/node_external_reference.h index c57a01ff39c..306c726631a 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -66,6 +66,7 @@ class ExternalReferenceRegistry { V(handle_wrap) \ V(heap_utils) \ V(messaging) \ + V(mksnapshot) \ V(native_module) \ V(options) \ V(os) \ diff --git a/src/node_options.cc b/src/node_options.cc index 3192faaddaf..f4b6f3884e4 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -716,6 +716,11 @@ PerProcessOptionsParser::PerProcessOptionsParser( "disable Object.prototype.__proto__", &PerProcessOptions::disable_proto, kAllowedInEnvironment); + AddOption("--build-snapshot", + "Generate a snapshot blob when the process exits." + "Currently only supported in the node_mksnapshot binary.", + &PerProcessOptions::build_snapshot, + kDisallowedInEnvironment); // 12.x renamed this inadvertently, so alias it for consistency within the // release line, while using the original name for consistency with older diff --git a/src/node_options.h b/src/node_options.h index 40d1c026058..39ab60ac254 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -224,6 +224,7 @@ class PerProcessOptions : public Options { bool zero_fill_all_buffers = false; bool debug_arraybuffer_allocations = false; std::string disable_proto; + bool build_snapshot; std::vector security_reverts; bool print_bash_completion = false; diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index 71b025df81e..327246743bb 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -18,12 +18,18 @@ namespace node { using v8::Context; +using v8::Function; +using v8::FunctionCallbackInfo; using v8::HandleScope; using v8::Isolate; using v8::Local; +using v8::MaybeLocal; using v8::Object; +using v8::ScriptCompiler; +using v8::ScriptOrigin; using v8::SnapshotCreator; using v8::StartupData; +using v8::String; using v8::TryCatch; using v8::Value; @@ -130,16 +136,36 @@ void SnapshotBuilder::Generate(SnapshotData* out, nullptr, node::EnvironmentFlags::kDefaultFlags, {}); + // TODO(joyeecheung): run env->InitializeInspector({}) here. // Run scripts in lib/internal/bootstrap/ { TryCatch bootstrapCatch(isolate); - v8::MaybeLocal result = env->RunBootstrapping(); + MaybeLocal result = env->RunBootstrapping(); if (bootstrapCatch.HasCaught()) { PrintCaughtException(isolate, context, bootstrapCatch); } result.ToLocalChecked(); } + // If --build-snapshot is true, lib/internal/main/mksnapshot.js would be + // loaded via LoadEnvironment() to execute process.argv[1] as the entry + // point (we currently only support this kind of entry point, but we + // could also explore snapshotting other kinds of execution modes + // in the future). + if (per_process::cli_options->build_snapshot) { + TryCatch bootstrapCatch(isolate); + // TODO(joyeecheung): we could use the result for something special, + // like setting up initializers that should be invoked at snapshot + // dehydration. + MaybeLocal result = + LoadEnvironment(env, StartExecutionCallback{}); + if (bootstrapCatch.HasCaught()) { + PrintCaughtException(isolate, context, bootstrapCatch); + } + result.ToLocalChecked(); + // TODO(joyeecheung): run SpinEventLoop here. + } + if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { env->PrintAllBaseObjects(); printf("Environment = %p\n", env); @@ -309,4 +335,57 @@ void SerializeBindingData(Environment* env, }); } +namespace mksnapshot { + +static void CompileSnapshotMain(const FunctionCallbackInfo& args) { + CHECK(args[0]->IsString()); + Local filename = args[0].As(); + Local source = args[1].As(); + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + ScriptOrigin origin(isolate, filename, 0, 0, true); + // TODO(joyeecheung): do we need all of these? Maybe we would want a less + // internal version of them. + std::vector> parameters = { + FIXED_ONE_BYTE_STRING(isolate, "require"), + FIXED_ONE_BYTE_STRING(isolate, "__filename"), + FIXED_ONE_BYTE_STRING(isolate, "__dirname"), + }; + ScriptCompiler::Source script_source(source, origin); + Local fn; + if (ScriptCompiler::CompileFunctionInContext(context, + &script_source, + parameters.size(), + parameters.data(), + 0, + nullptr, + ScriptCompiler::kEagerCompile) + .ToLocal(&fn)) { + args.GetReturnValue().Set(fn); + } +} + +static void Initialize(Local target, + Local unused, + Local context, + void* priv) { + Environment* env = Environment::GetCurrent(context); + Isolate* isolate = context->GetIsolate(); + env->SetMethod(target, "compileSnapshotMain", CompileSnapshotMain); + target + ->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "cleanups"), + v8::Array::New(isolate)) + .Check(); +} + +static void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(CompileSnapshotMain); + registry->Register(MarkBootstrapComplete); +} +} // namespace mksnapshot } // namespace node + +NODE_MODULE_CONTEXT_AWARE_INTERNAL(mksnapshot, node::mksnapshot::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(mksnapshot, + node::mksnapshot::RegisterExternalReferences) diff --git a/src/node_snapshotable.h b/src/node_snapshotable.h index 1ccd9a93226..30b684eb68a 100644 --- a/src/node_snapshotable.h +++ b/src/node_snapshotable.h @@ -127,6 +127,8 @@ class SnapshotBuilder { public: static std::string Generate(const std::vector args, const std::vector exec_args); + + // Generate the snapshot into out. static void Generate(SnapshotData* out, const std::vector args, const std::vector exec_args); diff --git a/tools/snapshot/node_mksnapshot.cc b/tools/snapshot/node_mksnapshot.cc index e591f64a2a0..60062854327 100644 --- a/tools/snapshot/node_mksnapshot.cc +++ b/tools/snapshot/node_mksnapshot.cc @@ -11,49 +11,87 @@ #include "util-inl.h" #include "v8.h" +int BuildSnapshot(int argc, char* argv[]); + #ifdef _WIN32 #include -int wmain(int argc, wchar_t* argv[]) { +int wmain(int argc, wchar_t* wargv[]) { + // Windows needs conversion from wchar_t to char. + + // Convert argv to UTF8. + char** argv = new char*[argc + 1]; + for (int i = 0; i < argc; i++) { + // Compute the size of the required buffer + DWORD size = WideCharToMultiByte( + CP_UTF8, 0, wargv[i], -1, nullptr, 0, nullptr, nullptr); + if (size == 0) { + // This should never happen. + fprintf(stderr, "Could not convert arguments to utf8."); + exit(1); + } + // Do the actual conversion + argv[i] = new char[size]; + DWORD result = WideCharToMultiByte( + CP_UTF8, 0, wargv[i], -1, argv[i], size, nullptr, nullptr); + if (result == 0) { + // This should never happen. + fprintf(stderr, "Could not convert arguments to utf8."); + exit(1); + } + } + argv[argc] = nullptr; #else // UNIX int main(int argc, char* argv[]) { argv = uv_setup_args(argc, argv); + + // Disable stdio buffering, it interacts poorly with printf() + // calls elsewhere in the program (e.g., any logging from V8.) + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); #endif // _WIN32 v8::V8::SetFlagsFromString("--random_seed=42"); + v8::V8::SetFlagsFromString("--harmony-import-assertions"); + return BuildSnapshot(argc, argv); +} +int BuildSnapshot(int argc, char* argv[]) { if (argc < 2) { std::cerr << "Usage: " << argv[0] << " \n"; + std::cerr << " " << argv[0] << " --build-snapshot " + << " \n"; return 1; } - std::ofstream out; - out.open(argv[1], std::ios::out | std::ios::binary); - if (!out.is_open()) { - std::cerr << "Cannot open " << argv[1] << "\n"; - return 1; - } - -// Windows needs conversion from wchar_t to char. See node_main.cc -#ifdef _WIN32 - int node_argc = 1; - char argv0[] = "node"; - char* node_argv[] = {argv0, nullptr}; - node::InitializationResult result = - node::InitializeOncePerProcess(node_argc, node_argv); -#else node::InitializationResult result = node::InitializeOncePerProcess(argc, argv); -#endif CHECK(!result.early_return); CHECK_EQ(result.exit_code, 0); + std::string out_path; + if (node::per_process::cli_options->build_snapshot) { + out_path = result.args[2]; + } else { + out_path = result.args[1]; + } + + std::ofstream out(out_path, std::ios::out | std::ios::binary); + if (!out) { + std::cerr << "Cannot open " << out_path << "\n"; + return 1; + } + { std::string snapshot = node::SnapshotBuilder::Generate(result.args, result.exec_args); out << snapshot; - out.close(); + + if (!out) { + std::cerr << "Failed to write " << out_path << "\n"; + return 1; + } } node::TearDownOncePerProcess();