feat: consume js inputFileSystem on the rust side

.github/actions/cache/restore/action.yml

@@ -15,7 +15,7 @@ inputs:
 
 outputs:
   cache-hit:
-    description: 'A boolean value to indicate an exact match was found for the primary key'
+    description: "A boolean value to indicate an exact match was found for the primary key"
     value: ${{ steps.github-cache.outputs.cache-hit == 'true' || steps.local-cache.outputs.cache-hit == 'true' }}
 
 runs:

crates/node_binding/binding.d.ts

@@ -281,7 +281,7 @@ export declare class JsCompilation {
 }
 
 export declare class JsCompiler {
-  constructor(compilerPath: string, options: RawOptions, builtinPlugins: Array<BuiltinPlugin>, registerJsTaps: RegisterJsTaps, outputFilesystem: ThreadsafeNodeFS, intermediateFilesystem: ThreadsafeNodeFS | undefined | null, resolverFactoryReference: JsResolverFactory)
+  constructor(compilerPath: string, options: RawOptions, builtinPlugins: Array<BuiltinPlugin>, registerJsTaps: RegisterJsTaps, outputFilesystem: ThreadsafeNodeFS, intermediateFilesystem: ThreadsafeNodeFS | undefined | null, inputFilesystem: ThreadsafeNodeFS | undefined | null, resolverFactoryReference: JsResolverFactory)
   setNonSkippableRegisters(kinds: Array<RegisterJsTapKind>): void
   /** Build with the given option passed to the constructor */
   build(callback: (err: null | Error) => void): void
@@ -1875,6 +1875,7 @@ incremental?: false | { [key: string]: boolean }
 parallelCodeSplitting: boolean
 rspackFuture?: RawRspackFuture
 cache: boolean | { type: "persistent" } & RawExperimentCacheOptionsPersistent | { type: "memory" }
+useInputFileSystem?: false | Array<RegExp>
 }
 
 export interface RawExperimentSnapshotOptions {
@@ -2712,6 +2713,7 @@ export interface ThreadsafeNodeFS {
   readFile: (name: string) => Promise<Buffer | string | void>
   stat: (name: string) => Promise<NodeFsStats | void>
   lstat: (name: string) => Promise<NodeFsStats | void>
+  realpath: (name: string) => Promise<string | void>
   open: (name: string, flags: string) => Promise<number | void>
   rename: (from: string, to: string) => Promise<void>
   close: (fd: number) => Promise<void>

crates/node_binding/src/fs_node/hybrid.rs

@@ -0,0 +1,71 @@
+use async_trait::async_trait;
+use rspack_fs::{FileMetadata, NativeFileSystem, ReadableFileSystem, Result};
+use rspack_paths::{Utf8Path, Utf8PathBuf};
+use rspack_regex::RspackRegex;
+
+use super::NodeFileSystem;
+
+#[derive(Debug)]
+pub struct HybridFileSystem {
+  allowlist: Vec<RspackRegex>,
+  node_fs: NodeFileSystem,
+  native_fs: NativeFileSystem,
+}
+
+impl HybridFileSystem {
+  pub fn new(
+    allowlist: Vec<RspackRegex>,
+    node_fs: NodeFileSystem,
+    native_fs: NativeFileSystem,
+  ) -> Self {
+    Self {
+      allowlist,
+      node_fs,
+      native_fs,
+    }
+  }
+
+  fn pick_fs_for_path(&self, path: &Utf8Path) -> &dyn ReadableFileSystem {
+    if self
+      .allowlist
+      .iter()
+      .any(|regexp| regexp.test(path.as_str()))
+    {
+      &self.node_fs
+    } else {
+      &self.native_fs
+    }
+  }
+}
+
+#[async_trait]
+impl ReadableFileSystem for HybridFileSystem {
+  async fn read(&self, path: &Utf8Path) -> Result<Vec<u8>> {
+    self.pick_fs_for_path(path).read(path).await
+  }
+  fn read_sync(&self, path: &Utf8Path) -> Result<Vec<u8>> {
+    self.pick_fs_for_path(path).read_sync(path)
+  }
+
+  async fn metadata(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    self.pick_fs_for_path(path).metadata(path).await
+  }
+  fn metadata_sync(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    self.pick_fs_for_path(path).metadata_sync(path)
+  }
+
+  async fn symlink_metadata(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    self.pick_fs_for_path(path).symlink_metadata(path).await
+  }
+
+  async fn canonicalize(&self, path: &Utf8Path) -> Result<Utf8PathBuf> {
+    self.pick_fs_for_path(path).canonicalize(path).await
+  }
+
+  async fn read_dir(&self, path: &Utf8Path) -> Result<Vec<String>> {
+    self.pick_fs_for_path(path).read_dir(path).await
+  }
+  fn read_dir_sync(&self, path: &Utf8Path) -> Result<Vec<String>> {
+    self.pick_fs_for_path(path).read_dir_sync(path)
+  }
+}

crates/node_binding/src/fs_node/mod.rs

@@ -5,3 +5,6 @@ pub use write::NodeFileSystem;
 
 mod node;
 pub use node::ThreadsafeNodeFS;
+
+mod hybrid;
+pub use hybrid::HybridFileSystem;

crates/node_binding/src/fs_node/node.rs

@@ -33,6 +33,8 @@ pub struct ThreadsafeNodeFS {
   pub stat: ThreadsafeFunction<String, Promise<Either<NodeFsStats, ()>>>,
   #[napi(ts_type = "(name: string) => Promise<NodeFsStats | void>")]
   pub lstat: ThreadsafeFunction<String, Promise<Either<NodeFsStats, ()>>>,
+  #[napi(ts_type = "(name: string) => Promise<string | void>")]
+  pub realpath: ThreadsafeFunction<String, Promise<Either<String, ()>>>,
   #[napi(ts_type = "(name: string, flags: string) => Promise<number | void>")]
   pub open: Open,
   #[napi(ts_type = "(from: string, to: string) => Promise<void>")]

crates/node_binding/src/fs_node/write.rs

@@ -1,12 +1,16 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use napi::{bindgen_prelude::Either3, Either};
+use napi::{
+  bindgen_prelude::{block_on, Either3},
+  Either,
+};
 use rspack_fs::{
-  Error, FileMetadata, IntermediateFileSystem, IntermediateFileSystemExtras, ReadStream, Result,
-  RspackResultToFsResultExt, WritableFileSystem, WriteStream,
+  Error, FileMetadata, IntermediateFileSystem, IntermediateFileSystemExtras, ReadStream,
+  ReadableFileSystem, Result, RspackResultToFsResultExt, WritableFileSystem, WriteStream,
 };
-use rspack_paths::Utf8Path;
+use rspack_paths::{Utf8Path, Utf8PathBuf};
+use tracing::instrument;
 
 use super::node::ThreadsafeNodeFS;
 
@@ -151,6 +155,106 @@ impl IntermediateFileSystemExtras for NodeFileSystem {
 
 impl IntermediateFileSystem for NodeFileSystem {}
 
+#[async_trait]
+impl ReadableFileSystem for NodeFileSystem {
+  #[instrument(skip(self), level = "debug")]
+  async fn read(&self, path: &Utf8Path) -> Result<Vec<u8>> {
+    self
+      .0
+      .read_file
+      .call_with_promise(path.as_str().to_string())
+      .await
+      .to_fs_result()
+      // TODO: simplify the return value?
+      .map(|result| match result {
+        Either3::A(buf) => buf.into(),
+        Either3::B(str) => str.into(),
+        Either3::C(_) => vec![],
+      })
+  }
+  #[instrument(skip(self), level = "debug")]
+  fn read_sync(&self, path: &Utf8Path) -> Result<Vec<u8>> {
+    block_on(self.read(path))
+  }
+
+  #[instrument(skip(self), level = "debug")]
+  async fn metadata(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    let res = self
+      .0
+      .stat
+      .call_with_promise(path.as_str().to_string())
+      .await
+      .to_fs_result()?;
+    match res {
+      Either::A(stats) => Ok(stats.into()),
+      Either::B(_) => Err(Error::new(
+        std::io::ErrorKind::Other,
+        "input file system call stat failed",
+      )),
+    }
+  }
+
+  #[instrument(skip(self), level = "debug")]
+  fn metadata_sync(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    block_on(self.metadata(path))
+  }
+
+  #[instrument(skip(self), level = "debug")]
+  async fn symlink_metadata(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    let res = self
+      .0
+      .lstat
+      .call_with_promise(path.as_str().to_string())
+      .await
+      .to_fs_result()?;
+    match res {
+      Either::A(stats) => Ok(stats.into()),
+      Either::B(_) => Err(Error::new(
+        std::io::ErrorKind::Other,
+        "input file system call lstat failed",
+      )),
+    }
+  }
+
+  #[instrument(skip(self), level = "debug")]
+  async fn canonicalize(&self, path: &Utf8Path) -> Result<Utf8PathBuf> {
+    let res = self
+      .0
+      .realpath
+      .call_with_promise(path.as_str().to_string())
+      .await
+      .to_fs_result()?;
+    match res {
+      Either::A(str) => Ok(Utf8PathBuf::from(str)),
+      Either::B(_) => Err(Error::new(
+        std::io::ErrorKind::Other,
+        "input file system call realpath failed",
+      )),
+    }
+  }
+
+  #[instrument(skip(self), level = "debug")]
+  async fn read_dir(&self, dir: &Utf8Path) -> Result<Vec<String>> {
+    let res = self
+      .0
+      .read_dir
+      .call_with_promise(dir.as_str().to_string())
+      .await
+      .to_fs_result()?;
+    match res {
+      Either::A(list) => Ok(list),
+      Either::B(_) => Err(Error::new(
+        std::io::ErrorKind::Other,
+        "input file system call read_dir failed",
+      )),
+    }
+  }
+  #[instrument(skip(self), level = "debug")]
+  fn read_dir_sync(&self, dir: &Utf8Path) -> Result<Vec<String>> {
+    block_on(ReadableFileSystem::read_dir(self, dir))
+  }
+}
+
 #[derive(Debug)]
 pub struct NodeReadStream {
   fd: i32,

crates/node_binding/src/lib.rs

@@ -9,13 +9,14 @@ extern crate rspack_allocator;
 use std::{cell::RefCell, sync::Arc};
 
 use compiler::{Compiler, CompilerState, CompilerStateGuard};
+use fs_node::HybridFileSystem;
 use napi::{bindgen_prelude::*, CallContext};
 use rspack_collections::UkeyMap;
 use rspack_core::{
   BoxDependency, Compilation, CompilerId, EntryOptions, ModuleIdentifier, PluginExt,
 };
 use rspack_error::Diagnostic;
-use rspack_fs::IntermediateFileSystem;
+use rspack_fs::{IntermediateFileSystem, NativeFileSystem, ReadableFileSystem};
 
 use crate::fs_node::{NodeFileSystem, ThreadsafeNodeFS};
 
@@ -140,11 +141,12 @@ impl JsCompiler {
     env: Env,
     mut this: This,
     compiler_path: String,
-    options: RawOptions,
+    mut options: RawOptions,
     builtin_plugins: Vec<BuiltinPlugin>,
     register_js_taps: RegisterJsTaps,
     output_filesystem: ThreadsafeNodeFS,
     intermediate_filesystem: Option<ThreadsafeNodeFS>,
+    input_filesystem: Option<ThreadsafeNodeFS>,
     mut resolver_factory_reference: Reference<JsResolverFactory>,
   ) -> Result<Self> {
     tracing::info!(name:"rspack_version", version = rspack_version!());
@@ -168,10 +170,36 @@ impl JsCompiler {
       bp.append_to(env, &mut this, &mut plugins)?;
     }
 
+    let use_input_fs = options.experiments.use_input_file_system.take();
     let compiler_options: rspack_core::CompilerOptions = options.try_into().to_napi_result()?;
 
     tracing::debug!(name:"normalized_options", options=?&compiler_options);
 
+    let input_file_system: Option<Arc<dyn ReadableFileSystem>> = input_filesystem.and_then(|fs| {
+      use_input_fs.and_then(|use_input_file_system| {
+        let node_fs = NodeFileSystem::new(fs).expect("Failed to create readable filesystem");
+
+        match use_input_file_system {
+          WithFalse::False => None,
+          WithFalse::True(allowlist) => {
+            if allowlist.is_empty() {
+              return None;
+            }
+            let binding: Arc<dyn ReadableFileSystem> = Arc::new(HybridFileSystem::new(
+              allowlist,
+              node_fs,
+              NativeFileSystem::new(compiler_options.resolve.pnp.unwrap_or(false)),
+            ));
+            Some(binding)
+          }
+        }
+      })
+    });
+
+    if let Some(fs) = &input_file_system {
+      resolver_factory_reference.input_filesystem = fs.clone();
+    }
+
     let resolver_factory =
       (*resolver_factory_reference).get_resolver_factory(compiler_options.resolve.clone());
     let loader_resolver_factory = (*resolver_factory_reference)
@@ -198,7 +226,7 @@ impl JsCompiler {
           .to_napi_result_with_message(|e| format!("Failed to create writable filesystem: {e}"))?,
       )),
       intermediate_filesystem,
-      None,
+      input_file_system,
       Some(resolver_factory),
       Some(loader_resolver_factory),
     );

crates/node_binding/src/raw_options/raw_experiments/mod.rs

@@ -7,6 +7,7 @@ use raw_cache::{normalize_raw_experiment_cache_options, RawExperimentCacheOption
 use raw_incremental::RawIncremental;
 use raw_rspack_future::RawRspackFuture;
 use rspack_core::{incremental::IncrementalOptions, Experiments};
+use rspack_regex::RspackRegex;
 
 use super::WithFalse;
 
@@ -23,6 +24,8 @@ pub struct RawExperiments {
     ts_type = r#"boolean | { type: "persistent" } & RawExperimentCacheOptionsPersistent | { type: "memory" }"#
   )]
   pub cache: RawExperimentCacheOptions,
+  #[napi(ts_type = "false | Array<RegExp>")]
+  pub use_input_file_system: Option<WithFalse<Vec<RspackRegex>>>,
 }
 
 impl From<RawExperiments> for Experiments {

crates/node_binding/src/resolver.rs

@@ -44,15 +44,21 @@ impl JsResolver {
     path: String,
     request: String,
   ) -> napi::Result<Either<JsResourceData, bool>> {
-    block_on(async move {
-      match self.resolver.resolve(Path::new(&path), &request).await {
-        Ok(rspack_core::ResolveResult::Resource(resource)) => {
-          Ok(Either::A(ResourceData::from(resource).into()))
-        }
-        Ok(rspack_core::ResolveResult::Ignored) => Ok(Either::B(false)),
-        Err(err) => Err(napi::Error::from_reason(format!("{err:?}"))),
+    block_on(self._resolve(path, request))
+  }
+
+  async fn _resolve(
+    &self,
+    path: String,
+    request: String,
+  ) -> napi::Result<Either<JsResourceData, bool>> {
+    match self.resolver.resolve(Path::new(&path), &request).await {
+      Ok(rspack_core::ResolveResult::Resource(resource)) => {
+        Ok(Either::A(ResourceData::from(resource).into()))
       }
-    })
+      Ok(rspack_core::ResolveResult::Ignored) => Ok(Either::B(false)),
+      Err(err) => Err(napi::Error::from_reason(format!("{err:?}"))),
+    }
   }
 
   #[napi(

packages/rspack-test-tools/src/helper/util/checkStats.js

@@ -11,7 +11,7 @@ exports.checkChunkModules = function checkChunkModules(
 		const chunkModules = chunk.modules.map(m => m.identifier);
 		if (strict && expectedModules.length !== chunkModules.length) {
 			throw new Error(
-				`expect chunk ${chunkId} has ${chunkModules.length} modules: ${chunkModules}\nbut received ${chunkModules.length} modules`
+				`expect chunk ${chunkId} has ${expectedModules.length} modules: ${expectedModules}\nbut received ${chunkModules.length} modules`
 			);
 		}
 

packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap

@@ -52,6 +52,7 @@ Object {
       },
     },
     topLevelAwait: true,
+    useInputFileSystem: false,
   },
   externals: undefined,
   externalsPresets: Object {

packages/rspack-test-tools/tests/configCases/input-file-system/disk.js

@@ -0,0 +1,3 @@
+it("should load a file from the disk", ()=>{
+	expect(42).toBe(42);
+})