refactor(turbopack-node): support execution by napi and worker_threads

Author: xusd320

Cargo.lock

@@ -423,6 +423,8 @@ napi = { version = "2", default-features = false, features = [
   "napi5",
   "compat-mode",
 ] }
+napi-derive = "2"
+napi-build = "2"
 notify = "8.1.0"
 once_cell = "1.17.1"
 owo-colors = "4.2.2"

Cargo.toml

@@ -46,12 +46,12 @@ workspace = true
 
 [package.metadata.cargo-shear]
 ignored = [
-  # we need to set features on these packages when building for WASM, but we don't directly use them
-  "getrandom",
-  "iana-time-zone",
-  # the plugins feature needs to set a feature on this transitively depended-on package, we never
-  # directly import it
-  "turbopack-ecmascript-plugins",
+    # we need to set features on these packages when building for WASM, but we don't directly use them
+    "getrandom",
+    "iana-time-zone",
+    # the plugins feature needs to set a feature on this transitively depended-on package, we never
+    # directly import it
+    "turbopack-ecmascript-plugins",
 ]
 
 [dependencies]
@@ -63,7 +63,7 @@ flate2 = { workspace = true }
 futures-util = { workspace = true }
 owo-colors = { workspace = true }
 napi = { workspace = true }
-napi-derive = "2"
+napi-derive = { workspace = true }
 next-custom-transforms = { workspace = true }
 next-taskless = { workspace = true }
 rand = { workspace = true }
@@ -115,7 +115,7 @@ next-core = { workspace = true }
 mdxjs = { workspace = true, features = ["serializable"] }
 
 turbo-tasks-malloc = { workspace = true, default-features = false, features = [
-    "custom_allocator"
+    "custom_allocator",
 ] }
 
 turbopack-core = { workspace = true }
@@ -143,8 +143,7 @@ tokio = { workspace = true, features = ["full"] }
 
 [build-dependencies]
 anyhow = { workspace = true }
-napi-build = "2"
+napi-build = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 vergen-gitcl = { workspace = true }
-

crates/napi/Cargo.toml

@@ -1057,11 +1057,12 @@ impl Project {
 
             // At this point all modules have been computed and we can get rid of the node.js
             // process pools
-            if *self.is_watch_enabled().await? {
-                turbopack_node::evaluate::scale_down();
-            } else {
-                turbopack_node::evaluate::scale_zero();
-            }
+            // TODO:
+            // if *self.is_watch_enabled().await? {
+            //     turbopack_node::evaluate::scale_down();
+            // } else {
+            //     turbopack_node::evaluate::scale_zero();
+            // }
 
             Ok(module_graphs_vc)
         }

crates/next-api/src/project.rs

@@ -34,6 +34,18 @@ export declare class ExternalObject<T> {
     [K: symbol]: T
   }
 }
+export declare function recvPoolRequest(): Promise<string>
+export declare function notifyPoolCreated(filename: string): Promise<void>
+export declare function recvWorkerRequest(poolId: string): Promise<void>
+export declare function notifyWorkerAck(poolId: string): Promise<void>
+export declare function recvEvaluation(poolId: string): Promise<Array<number>>
+export declare function recvMessageInWorker(
+  workerId: number
+): Promise<Array<number>>
+export declare function sendTaskResponse(
+  taskId: string,
+  data: Array<number>
+): Promise<void>
 export declare function lockfileTryAcquireSync(
   path: string
 ): { __napiType: 'Lockfile' } | null

packages/next/src/build/swc/generated-native.d.ts

@@ -10,8 +10,11 @@ autobenches = false
 bench = false
 
 [features]
+default = ["worker_thread"]
 # enable "HMR" for embedded assets
 dynamic_embed_contents = ["turbo-tasks-fs/dynamic_embed_contents"]
+child_process = ["tokio"]
+worker_thread = ["napi", "napi-derive"]
 
 [lints]
 workspace = true
@@ -20,6 +23,8 @@ workspace = true
 anyhow = { workspace = true }
 async-stream = "0.3.4"
 async-trait = { workspace = true }
+async-channel = "2.5.0"
+dashmap = { workspace = true }
 base64 = "0.21.0"
 const_format = { workspace = true }
 either = { workspace = true, features = ["serde"] }
@@ -34,7 +39,10 @@ rustc-hash = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_with = { workspace = true, features = ["base64"] }
-tokio = { workspace = true, features = ["full"] }
+tokio = { workspace = true, optional = true, features = ["full"] }
+uuid = { workspace = true, features = ["v4"] }
+napi = { workspace = true, optional = true, features = ["anyhow"] }
+napi-derive = { workspace = true, optional = true }
 tracing = { workspace = true }
 turbo-rcstr = { workspace = true }
 turbo-tasks = { workspace = true }
@@ -46,3 +54,5 @@ turbopack-core = { workspace = true }
 turbopack-ecmascript = { workspace = true }
 turbopack-resolve = { workspace = true }
 
+[build-dependencies]
+napi-build = { workspace = true }

turbopack/crates/turbopack-node/Cargo.toml

@@ -0,0 +1,6 @@
+fn main() {
+    #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
+    if std::env::var("CARGO_FEATURE_WORKER_THREAD").is_ok() {
+        napi_build::setup();
+    }
+}