src: support configurable snapshot

- Add support for --build-snapshot-config which allows passing
  snapshot configurations via a JSON configuration file.
- Add support for node::SnapshotConfig in the embedder API

The initial configurable options are:

- "builder" (SnapshotConfig::builder_script_path): path to the
  builder script.
- "withoutCodeCache" (SnapshotFlags::kWithoutCodeCache): disable
  code cache generation.

src/api/embed_helpers.cc

@@ -92,7 +92,8 @@ CommonEnvironmentSetup::CommonEnvironmentSetup(
     std::vector<std::string>* errors,
     const EmbedderSnapshotData* snapshot_data,
     uint32_t flags,
-    std::function<Environment*(const CommonEnvironmentSetup*)> make_env)
+    std::function<Environment*(const CommonEnvironmentSetup*)> make_env,
+    const SnapshotConfig* snapshot_config)
     : impl_(new Impl()) {
   CHECK_NOT_NULL(platform);
   CHECK_NOT_NULL(errors);
@@ -142,8 +143,7 @@ CommonEnvironmentSetup::CommonEnvironmentSetup(
 
     impl_->isolate_data.reset(CreateIsolateData(
         isolate, loop, platform, impl_->allocator.get(), snapshot_data));
-    impl_->isolate_data->set_is_building_snapshot(
-        impl_->snapshot_creator.has_value());
+    impl_->isolate_data->set_snapshot_config(snapshot_config);
 
     if (snapshot_data) {
       impl_->env.reset(make_env(this));
@@ -176,7 +176,8 @@ CommonEnvironmentSetup::CreateForSnapshotting(
     MultiIsolatePlatform* platform,
     std::vector<std::string>* errors,
     const std::vector<std::string>& args,
-    const std::vector<std::string>& exec_args) {
+    const std::vector<std::string>& exec_args,
+    const SnapshotConfig& snapshot_config) {
   // It's not guaranteed that a context that goes through
   // v8_inspector::V8Inspector::contextCreated() is runtime-independent,
   // so do not start the inspector on the main context when building
@@ -196,7 +197,8 @@ CommonEnvironmentSetup::CreateForSnapshotting(
             args,
             exec_args,
             static_cast<EnvironmentFlags::Flags>(env_flags));
-      }));
+      },
+      &snapshot_config));
   if (!errors->empty()) ret.reset();
   return ret;
 }
@@ -240,10 +242,7 @@ EmbedderSnapshotData::Pointer CommonEnvironmentSetup::CreateSnapshot() {
   EmbedderSnapshotData::Pointer result{
       new EmbedderSnapshotData(snapshot_data, true)};
 
-  auto exit_code = SnapshotBuilder::CreateSnapshot(
-      snapshot_data,
-      this,
-      static_cast<uint8_t>(SnapshotMetadata::Type::kFullyCustomized));
+  auto exit_code = SnapshotBuilder::CreateSnapshot(snapshot_data, this);
   if (exit_code != ExitCode::kNoFailure) return {};
 
   return result;

src/env.cc

@@ -285,6 +285,12 @@ std::ostream& operator<<(std::ostream& output,
   return output;
 }
 
+std::ostream& operator<<(std::ostream& output, const SnapshotFlags& flags) {
+  output << "static_cast<SnapshotFlags>(" << static_cast<uint32_t>(flags)
+         << ")";
+  return output;
+}
+
 std::ostream& operator<<(std::ostream& output, const SnapshotMetadata& i) {
   output << "{\n"
          << "  "
@@ -296,6 +302,7 @@ std::ostream& operator<<(std::ostream& output, const SnapshotMetadata& i) {
          << "  \"" << i.node_arch << "\", // node_arch\n"
          << "  \"" << i.node_platform << "\", // node_platform\n"
          << "  " << i.v8_cache_version_tag << ", // v8_cache_version_tag\n"
+         << "  " << i.flags << ", // flags\n"
          << "}";
   return output;
 }
@@ -802,8 +809,14 @@ Environment::Environment(IsolateData* isolate_data,
         isolate_data->worker_context()->env()->builtin_loader());
   } else if (isolate_data->snapshot_data() != nullptr) {
     // ... otherwise, if a snapshot was provided, use its code cache.
-    builtin_loader()->RefreshCodeCache(
-        isolate_data->snapshot_data()->code_cache);
+    size_t cache_size = isolate_data->snapshot_data()->code_cache.size();
+    per_process::Debug(DebugCategory::CODE_CACHE,
+                       "snapshot contains %zu code cache\n",
+                       cache_size);
+    if (cache_size > 0) {
+      builtin_loader()->RefreshCodeCache(
+          isolate_data->snapshot_data()->code_cache);
+    }
   }
 
   // We'll be creating new objects so make sure we've entered the context.

src/env.h

@@ -147,8 +147,15 @@ class NODE_EXTERN_PRIVATE IsolateData : public MemoryRetainer {
   void MemoryInfo(MemoryTracker* tracker) const override;
   IsolateDataSerializeInfo Serialize(v8::SnapshotCreator* creator);
 
-  bool is_building_snapshot() const { return is_building_snapshot_; }
-  void set_is_building_snapshot(bool value) { is_building_snapshot_ = value; }
+  bool is_building_snapshot() const { return snapshot_config_.has_value(); }
+  const SnapshotConfig* snapshot_config() const {
+    return snapshot_config_.has_value() ? &(snapshot_config_.value()) : nullptr;
+  }
+  void set_snapshot_config(const SnapshotConfig* config) {
+    if (config != nullptr) {
+      snapshot_config_ = *config;  // Copy the config.
+    }
+  }
 
   uint16_t* embedder_id_for_cppgc() const;
   uint16_t* embedder_id_for_non_cppgc() const;
@@ -237,11 +244,13 @@ class NODE_EXTERN_PRIVATE IsolateData : public MemoryRetainer {
   uv_loop_t* const event_loop_;
   NodeArrayBufferAllocator* const node_allocator_;
   MultiIsolatePlatform* platform_;
+
   const SnapshotData* snapshot_data_;
+  std::optional<SnapshotConfig> snapshot_config_;
+
   std::unique_ptr<v8::CppHeap> cpp_heap_;
   std::shared_ptr<PerIsolateOptions> options_;
   worker::Worker* worker_context_ = nullptr;
-  bool is_building_snapshot_ = false;
   PerIsolateWrapperData* wrapper_data_;
 
   static Mutex isolate_data_mutex_;
@@ -526,6 +535,7 @@ struct SnapshotMetadata {
   std::string node_platform;
   // Result of v8::ScriptCompiler::CachedDataVersionTag().
   uint32_t v8_cache_version_tag;
+  SnapshotFlags flags;
 };
 
 struct SnapshotData {

src/node.cc

@@ -1204,10 +1204,39 @@ ExitCode GenerateAndWriteSnapshotData(const SnapshotData** snapshot_data_ptr,
   // nullptr indicates there's no snapshot data.
   DCHECK_NULL(*snapshot_data_ptr);
 
+  SnapshotConfig snapshot_config;
+  const std::string& config_path =
+      per_process::cli_options->per_isolate->build_snapshot_config;
+  // For snapshot config read from JSON, we fix up process.argv[1] using the
+  // "builder" field.
+  std::vector<std::string> args_maybe_patched;
+  args_maybe_patched.reserve(result->args().size() + 1);
+  if (!config_path.empty()) {
+    std::optional<SnapshotConfig> optional_config =
+        ReadSnapshotConfig(config_path.c_str());
+    if (!optional_config.has_value()) {
+      return ExitCode::kGenericUserError;
+    }
+    snapshot_config = std::move(optional_config.value());
+    DCHECK(snapshot_config.builder_script_path.has_value());
+    args_maybe_patched.emplace_back(result->args()[0]);
+    args_maybe_patched.emplace_back(
+        snapshot_config.builder_script_path.value());
+    if (result->args().size() > 1) {
+      args_maybe_patched.insert(args_maybe_patched.end(),
+                                result->args().begin() + 1,
+                                result->args().end());
+    }
+  } else {
+    snapshot_config.builder_script_path = result->args()[1];
+    args_maybe_patched = result->args();
+  }
+  DCHECK(snapshot_config.builder_script_path.has_value());
+  const std::string& builder_script =
+      snapshot_config.builder_script_path.value();
   // node:embedded_snapshot_main indicates that we are using the
   // embedded snapshot and we are not supposed to clean it up.
-  const std::string& main_script = result->args()[1];
-  if (main_script == "node:embedded_snapshot_main") {
+  if (builder_script == "node:embedded_snapshot_main") {
     *snapshot_data_ptr = SnapshotBuilder::GetEmbeddedSnapshotData();
     if (*snapshot_data_ptr == nullptr) {
       // The Node.js binary is built without embedded snapshot
@@ -1219,24 +1248,25 @@ ExitCode GenerateAndWriteSnapshotData(const SnapshotData** snapshot_data_ptr,
       return exit_code;
     }
   } else {
-    // Otherwise, load and run the specified main script.
+    // Otherwise, load and run the specified builder script.
     std::unique_ptr<SnapshotData> generated_data =
         std::make_unique<SnapshotData>();
-    std::string main_script_content;
-    int r = ReadFileSync(&main_script_content, main_script.c_str());
+    std::string builder_script_content;
+    int r = ReadFileSync(&builder_script_content, builder_script.c_str());
     if (r != 0) {
       FPrintF(stderr,
-              "Cannot read main script %s for building snapshot. %s: %s",
-              main_script,
+              "Cannot read builder script %s for building snapshot. %s: %s",
+              builder_script,
               uv_err_name(r),
               uv_strerror(r));
       return ExitCode::kGenericUserError;
     }
 
     exit_code = node::SnapshotBuilder::Generate(generated_data.get(),
-                                                result->args(),
+                                                args_maybe_patched,
                                                 result->exec_args(),
-                                                main_script_content);
+                                                builder_script_content,
+                                                snapshot_config);
     if (exit_code == ExitCode::kNoFailure) {
       *snapshot_data_ptr = generated_data.release();
     } else {
@@ -1366,7 +1396,8 @@ static ExitCode StartInternal(int argc, char** argv) {
 
   // --build-snapshot indicates that we are in snapshot building mode.
   if (per_process::cli_options->per_isolate->build_snapshot) {
-    if (result->args().size() < 2) {
+    if (per_process::cli_options->per_isolate->build_snapshot_config.empty() &&
+        result->args().size() < 2) {
       fprintf(stderr,
               "--build-snapshot must be used with an entry point script.\n"
               "Usage: node --build-snapshot /path/to/entry.js\n");

src/node.h

@@ -80,6 +80,7 @@
 
 #include <functional>
 #include <memory>
+#include <optional>
 #include <ostream>
 
 // We cannot use __POSIX__ in this header because that's only defined when
@@ -659,6 +660,33 @@ enum Flags : uint64_t {
 };
 }  // namespace EnvironmentFlags
 
+enum class SnapshotFlags : uint32_t {
+  kDefault = 0,
+  // Whether code cache should be generated as part of the snapshot.
+  // Code cache reduce the time spent on compiling functions included
+  // in the snapshot at the expense of a bigger snapshot size and
+  // potentially breaking portability of the snapshot.
+  kWithoutCodeCache = 1 << 0,
+};
+
+struct SnapshotConfig {
+  SnapshotFlags flags = SnapshotFlags::kDefault;
+
+  // When builder_script_path is std::nullopt, the snapshot is generated as a
+  // built-in snapshot instead of a custom one, and it's expected that the
+  // built-in snapshot only contains states that reproduce in every run of the
+  // application. The event loop won't be run when generating a built-in
+  // snapshot, so asynchronous operations should be avoided.
+  //
+  // When builder_script_path is an std::string, it should match args[1]
+  // passed to CreateForSnapshotting(). The embedder is also expected to use
+  // LoadEnvironment() to run a script matching this path. In that case the
+  // snapshot is generated as a custom snapshot and the event loop is run, so
+  // the snapshot builder can execute asynchronous operations as long as they
+  // are run to completion when the snasphot is taken.
+  std::optional<std::string> builder_script_path;
+};
+
 struct InspectorParentHandle {
   virtual ~InspectorParentHandle() = default;
 };
@@ -870,7 +898,8 @@ class NODE_EXTERN CommonEnvironmentSetup {
       MultiIsolatePlatform* platform,
       std::vector<std::string>* errors,
       const std::vector<std::string>& args = {},
-      const std::vector<std::string>& exec_args = {});
+      const std::vector<std::string>& exec_args = {},
+      const SnapshotConfig& snapshot_config = {});
   EmbedderSnapshotData::Pointer CreateSnapshot();
 
   struct uv_loop_s* event_loop() const;
@@ -905,7 +934,8 @@ class NODE_EXTERN CommonEnvironmentSetup {
       std::vector<std::string>*,
       const EmbedderSnapshotData*,
       uint32_t flags,
-      std::function<Environment*(const CommonEnvironmentSetup*)>);
+      std::function<Environment*(const CommonEnvironmentSetup*)>,
+      const SnapshotConfig* config = nullptr);
 };
 
 // Implementation for CommonEnvironmentSetup::Create

src/node_internals.h

@@ -408,6 +408,7 @@ std::string Basename(const std::string& str, const std::string& extension);
 
 node_module napi_module_to_node_module(const napi_module* mod);
 
+std::ostream& operator<<(std::ostream& output, const SnapshotFlags& flags);
 std::ostream& operator<<(std::ostream& output,
                          const std::vector<SnapshotIndex>& v);
 std::ostream& operator<<(std::ostream& output,

src/node_main_instance.cc

@@ -56,8 +56,6 @@ NodeMainInstance::NodeMainInstance(const SnapshotData* snapshot_data,
                         platform,
                         array_buffer_allocator_.get(),
                         snapshot_data->AsEmbedderWrapper().get()));
-  isolate_data_->set_is_building_snapshot(
-      per_process::cli_options->per_isolate->build_snapshot);
 
   isolate_data_->max_young_gen_size =
       isolate_params_->constraints.max_young_generation_size_in_bytes();

src/node_options.cc

@@ -838,6 +838,12 @@ PerIsolateOptionsParser::PerIsolateOptionsParser(
             "Generate a snapshot blob when the process exits.",
             &PerIsolateOptions::build_snapshot,
             kDisallowedInEnvvar);
+  AddOption("--build-snapshot-config",
+            "Generate a snapshot blob when the process exits using a"
+            "JSON configuration in the specified path.",
+            &PerIsolateOptions::build_snapshot_config,
+            kDisallowedInEnvvar);
+  Implies("--build-snapshot-config", "--build-snapshot");
 
   Insert(eop, &PerIsolateOptions::get_per_env_options);
 }

src/node_options.h

@@ -232,6 +232,7 @@ class PerIsolateOptions : public Options {
   bool experimental_shadow_realm = false;
   std::string report_signal = "SIGUSR2";
   bool build_snapshot = false;
+  std::string build_snapshot_config;
   inline EnvironmentOptions* get_per_env_options();
   void CheckOptions(std::vector<std::string>* errors,
                     std::vector<std::string>* argv) override;

src/node_sea.cc

@@ -377,14 +377,18 @@ std::optional<SeaConfig> ParseSingleExecutableConfig(
 ExitCode GenerateSnapshotForSEA(const SeaConfig& config,
                                 const std::vector<std::string>& args,
                                 const std::vector<std::string>& exec_args,
-                                const std::string& main_script,
+                                const std::string& builder_script_content,
+                                const SnapshotConfig& snapshot_config,
                                 std::vector<char>* snapshot_blob) {
   SnapshotData snapshot;
   // TODO(joyeecheung): make the arguments configurable through the JSON
   // config or a programmatic API.
   std::vector<std::string> patched_args = {args[0], config.main_path};
-  ExitCode exit_code = SnapshotBuilder::Generate(
-      &snapshot, patched_args, exec_args, main_script);
+  ExitCode exit_code = SnapshotBuilder::Generate(&snapshot,
+                                                 patched_args,
+                                                 exec_args,
+                                                 builder_script_content,
+                                                 snapshot_config);
   if (exit_code != ExitCode::kNoFailure) {
     return exit_code;
   }
@@ -479,8 +483,11 @@ ExitCode GenerateSingleExecutableBlob(
   bool builds_snapshot_from_main =
       static_cast<bool>(config.flags & SeaFlags::kUseSnapshot);
   if (builds_snapshot_from_main) {
+    // TODO(joyeecheung): allow passing snapshot configuration in SEA configs.
+    SnapshotConfig snapshot_config;
+    snapshot_config.builder_script_path = main_script;
     ExitCode exit_code = GenerateSnapshotForSEA(
-        config, args, exec_args, main_script, &snapshot_blob);
+        config, args, exec_args, main_script, snapshot_config, &snapshot_blob);
     if (exit_code != ExitCode::kNoFailure) {
       return exit_code;
     }

src/node_snapshot_builder.h

@@ -16,20 +16,25 @@ namespace node {
 class ExternalReferenceRegistry;
 struct SnapshotData;
 
+std::optional<SnapshotConfig> ReadSnapshotConfig(const char* path);
+
 class NODE_EXTERN_PRIVATE SnapshotBuilder {
  public:
-  static ExitCode GenerateAsSource(
-      const char* out_path,
+  static ExitCode GenerateAsSource(const char* out_path,
+                                   const std::vector<std::string>& args,
+                                   const std::vector<std::string>& exec_args,
+                                   const SnapshotConfig& config,
+                                   bool use_array_literals = false);
+
+  // Generate the snapshot into out. builder_script_content should match
+  // config.builder_script_path. This is passed separately
+  // in case the script is already read for other purposes.
+  static ExitCode Generate(
+      SnapshotData* out,
       const std::vector<std::string>& args,
       const std::vector<std::string>& exec_args,
-      std::optional<std::string_view> main_script_path = std::nullopt,
-      bool use_array_literals = false);
-
-  // Generate the snapshot into out.
-  static ExitCode Generate(SnapshotData* out,
-                           const std::vector<std::string>& args,
-                           const std::vector<std::string>& exec_args,
-                           std::optional<std::string_view> main_script);
+      std::optional<std::string_view> builder_script_content,
+      const SnapshotConfig& config);
 
   // If nullptr is returned, the binary is not built with embedded
   // snapshot.
@@ -39,10 +44,8 @@ class NODE_EXTERN_PRIVATE SnapshotBuilder {
 
   static const std::vector<intptr_t>& CollectExternalReferences();
 
-  static ExitCode CreateSnapshot(
-      SnapshotData* out,
-      CommonEnvironmentSetup* setup,
-      /*SnapshotMetadata::Type*/ uint8_t snapshot_type);
+  static ExitCode CreateSnapshot(SnapshotData* out,
+                                 CommonEnvironmentSetup* setup);
 
  private:
   static std::unique_ptr<ExternalReferenceRegistry> registry_;