diff --git a/.github/actions/cache/restore/action.yml b/.github/actions/cache/restore/action.yml index ae1840a3ab3b..29ab0224d43f 100644 --- a/.github/actions/cache/restore/action.yml +++ b/.github/actions/cache/restore/action.yml @@ -15,7 +15,7 @@ inputs: outputs: cache-hit: - description: 'A boolean value to indicate an exact match was found for the primary key' + description: "A boolean value to indicate an exact match was found for the primary key" value: ${{ steps.github-cache.outputs.cache-hit == 'true' || steps.local-cache.outputs.cache-hit == 'true' }} runs: diff --git a/crates/node_binding/binding.d.ts b/crates/node_binding/binding.d.ts index d21b49fef0b3..56002e5b660a 100644 --- a/crates/node_binding/binding.d.ts +++ b/crates/node_binding/binding.d.ts @@ -281,7 +281,7 @@ export declare class JsCompilation { } export declare class JsCompiler { - constructor(compilerPath: string, options: RawOptions, builtinPlugins: Array, registerJsTaps: RegisterJsTaps, outputFilesystem: ThreadsafeNodeFS, intermediateFilesystem: ThreadsafeNodeFS | undefined | null, resolverFactoryReference: JsResolverFactory) + constructor(compilerPath: string, options: RawOptions, builtinPlugins: Array, registerJsTaps: RegisterJsTaps, outputFilesystem: ThreadsafeNodeFS, intermediateFilesystem: ThreadsafeNodeFS | undefined | null, inputFilesystem: ThreadsafeNodeFS | undefined | null, resolverFactoryReference: JsResolverFactory) setNonSkippableRegisters(kinds: Array): void /** Build with the given option passed to the constructor */ build(callback: (err: null | Error) => void): void @@ -1875,6 +1875,7 @@ incremental?: false | { [key: string]: boolean } parallelCodeSplitting: boolean rspackFuture?: RawRspackFuture cache: boolean | { type: "persistent" } & RawExperimentCacheOptionsPersistent | { type: "memory" } +useInputFileSystem?: false | Array } export interface RawExperimentSnapshotOptions { @@ -2712,6 +2713,7 @@ export interface ThreadsafeNodeFS { readFile: (name: string) => Promise stat: (name: string) => Promise lstat: (name: string) => Promise + realpath: (name: string) => Promise open: (name: string, flags: string) => Promise rename: (from: string, to: string) => Promise close: (fd: number) => Promise diff --git a/crates/node_binding/src/fs_node/hybrid.rs b/crates/node_binding/src/fs_node/hybrid.rs new file mode 100644 index 000000000000..a8a9c05c5a56 --- /dev/null +++ b/crates/node_binding/src/fs_node/hybrid.rs @@ -0,0 +1,71 @@ +use async_trait::async_trait; +use rspack_fs::{FileMetadata, NativeFileSystem, ReadableFileSystem, Result}; +use rspack_paths::{Utf8Path, Utf8PathBuf}; +use rspack_regex::RspackRegex; + +use super::NodeFileSystem; + +#[derive(Debug)] +pub struct HybridFileSystem { + allowlist: Vec, + node_fs: NodeFileSystem, + native_fs: NativeFileSystem, +} + +impl HybridFileSystem { + pub fn new( + allowlist: Vec, + node_fs: NodeFileSystem, + native_fs: NativeFileSystem, + ) -> Self { + Self { + allowlist, + node_fs, + native_fs, + } + } + + fn pick_fs_for_path(&self, path: &Utf8Path) -> &dyn ReadableFileSystem { + if self + .allowlist + .iter() + .any(|regexp| regexp.test(path.as_str())) + { + &self.node_fs + } else { + &self.native_fs + } + } +} + +#[async_trait] +impl ReadableFileSystem for HybridFileSystem { + async fn read(&self, path: &Utf8Path) -> Result> { + self.pick_fs_for_path(path).read(path).await + } + fn read_sync(&self, path: &Utf8Path) -> Result> { + self.pick_fs_for_path(path).read_sync(path) + } + + async fn metadata(&self, path: &Utf8Path) -> Result { + self.pick_fs_for_path(path).metadata(path).await + } + fn metadata_sync(&self, path: &Utf8Path) -> Result { + self.pick_fs_for_path(path).metadata_sync(path) + } + + async fn symlink_metadata(&self, path: &Utf8Path) -> Result { + self.pick_fs_for_path(path).symlink_metadata(path).await + } + + async fn canonicalize(&self, path: &Utf8Path) -> Result { + self.pick_fs_for_path(path).canonicalize(path).await + } + + async fn read_dir(&self, path: &Utf8Path) -> Result> { + self.pick_fs_for_path(path).read_dir(path).await + } + fn read_dir_sync(&self, path: &Utf8Path) -> Result> { + self.pick_fs_for_path(path).read_dir_sync(path) + } +} diff --git a/crates/node_binding/src/fs_node/mod.rs b/crates/node_binding/src/fs_node/mod.rs index e7429ea28c1f..ba8c3a258175 100644 --- a/crates/node_binding/src/fs_node/mod.rs +++ b/crates/node_binding/src/fs_node/mod.rs @@ -5,3 +5,6 @@ pub use write::NodeFileSystem; mod node; pub use node::ThreadsafeNodeFS; + +mod hybrid; +pub use hybrid::HybridFileSystem; diff --git a/crates/node_binding/src/fs_node/node.rs b/crates/node_binding/src/fs_node/node.rs index 2b78b28d49f4..ccbe8e3e881a 100644 --- a/crates/node_binding/src/fs_node/node.rs +++ b/crates/node_binding/src/fs_node/node.rs @@ -33,6 +33,8 @@ pub struct ThreadsafeNodeFS { pub stat: ThreadsafeFunction>>, #[napi(ts_type = "(name: string) => Promise")] pub lstat: ThreadsafeFunction>>, + #[napi(ts_type = "(name: string) => Promise")] + pub realpath: ThreadsafeFunction>>, #[napi(ts_type = "(name: string, flags: string) => Promise")] pub open: Open, #[napi(ts_type = "(from: string, to: string) => Promise")] diff --git a/crates/node_binding/src/fs_node/write.rs b/crates/node_binding/src/fs_node/write.rs index 6d9f6cf089bb..fe6870eab229 100644 --- a/crates/node_binding/src/fs_node/write.rs +++ b/crates/node_binding/src/fs_node/write.rs @@ -1,12 +1,16 @@ use std::sync::Arc; use async_trait::async_trait; -use napi::{bindgen_prelude::Either3, Either}; +use napi::{ + bindgen_prelude::{block_on, Either3}, + Either, +}; use rspack_fs::{ - Error, FileMetadata, IntermediateFileSystem, IntermediateFileSystemExtras, ReadStream, Result, - RspackResultToFsResultExt, WritableFileSystem, WriteStream, + Error, FileMetadata, IntermediateFileSystem, IntermediateFileSystemExtras, ReadStream, + ReadableFileSystem, Result, RspackResultToFsResultExt, WritableFileSystem, WriteStream, }; -use rspack_paths::Utf8Path; +use rspack_paths::{Utf8Path, Utf8PathBuf}; +use tracing::instrument; use super::node::ThreadsafeNodeFS; @@ -151,6 +155,106 @@ impl IntermediateFileSystemExtras for NodeFileSystem { impl IntermediateFileSystem for NodeFileSystem {} +#[async_trait] +impl ReadableFileSystem for NodeFileSystem { + #[instrument(skip(self), level = "debug")] + async fn read(&self, path: &Utf8Path) -> Result> { + self + .0 + .read_file + .call_with_promise(path.as_str().to_string()) + .await + .to_fs_result() + // TODO: simplify the return value? + .map(|result| match result { + Either3::A(buf) => buf.into(), + Either3::B(str) => str.into(), + Either3::C(_) => vec![], + }) + } + #[instrument(skip(self), level = "debug")] + fn read_sync(&self, path: &Utf8Path) -> Result> { + block_on(self.read(path)) + } + + #[instrument(skip(self), level = "debug")] + async fn metadata(&self, path: &Utf8Path) -> Result { + let res = self + .0 + .stat + .call_with_promise(path.as_str().to_string()) + .await + .to_fs_result()?; + match res { + Either::A(stats) => Ok(stats.into()), + Either::B(_) => Err(Error::new( + std::io::ErrorKind::Other, + "input file system call stat failed", + )), + } + } + + #[instrument(skip(self), level = "debug")] + fn metadata_sync(&self, path: &Utf8Path) -> Result { + block_on(self.metadata(path)) + } + + #[instrument(skip(self), level = "debug")] + async fn symlink_metadata(&self, path: &Utf8Path) -> Result { + let res = self + .0 + .lstat + .call_with_promise(path.as_str().to_string()) + .await + .to_fs_result()?; + match res { + Either::A(stats) => Ok(stats.into()), + Either::B(_) => Err(Error::new( + std::io::ErrorKind::Other, + "input file system call lstat failed", + )), + } + } + + #[instrument(skip(self), level = "debug")] + async fn canonicalize(&self, path: &Utf8Path) -> Result { + let res = self + .0 + .realpath + .call_with_promise(path.as_str().to_string()) + .await + .to_fs_result()?; + match res { + Either::A(str) => Ok(Utf8PathBuf::from(str)), + Either::B(_) => Err(Error::new( + std::io::ErrorKind::Other, + "input file system call realpath failed", + )), + } + } + + #[instrument(skip(self), level = "debug")] + async fn read_dir(&self, dir: &Utf8Path) -> Result> { + let res = self + .0 + .read_dir + .call_with_promise(dir.as_str().to_string()) + .await + .to_fs_result()?; + match res { + Either::A(list) => Ok(list), + Either::B(_) => Err(Error::new( + std::io::ErrorKind::Other, + "input file system call read_dir failed", + )), + } + } + #[instrument(skip(self), level = "debug")] + fn read_dir_sync(&self, dir: &Utf8Path) -> Result> { + block_on(ReadableFileSystem::read_dir(self, dir)) + } +} + #[derive(Debug)] pub struct NodeReadStream { fd: i32, diff --git a/crates/node_binding/src/lib.rs b/crates/node_binding/src/lib.rs index adf1f1bd080c..c59e287ba79a 100644 --- a/crates/node_binding/src/lib.rs +++ b/crates/node_binding/src/lib.rs @@ -9,13 +9,14 @@ extern crate rspack_allocator; use std::{cell::RefCell, sync::Arc}; use compiler::{Compiler, CompilerState, CompilerStateGuard}; +use fs_node::HybridFileSystem; use napi::{bindgen_prelude::*, CallContext}; use rspack_collections::UkeyMap; use rspack_core::{ BoxDependency, Compilation, CompilerId, EntryOptions, ModuleIdentifier, PluginExt, }; use rspack_error::Diagnostic; -use rspack_fs::IntermediateFileSystem; +use rspack_fs::{IntermediateFileSystem, NativeFileSystem, ReadableFileSystem}; use crate::fs_node::{NodeFileSystem, ThreadsafeNodeFS}; @@ -140,11 +141,12 @@ impl JsCompiler { env: Env, mut this: This, compiler_path: String, - options: RawOptions, + mut options: RawOptions, builtin_plugins: Vec, register_js_taps: RegisterJsTaps, output_filesystem: ThreadsafeNodeFS, intermediate_filesystem: Option, + input_filesystem: Option, mut resolver_factory_reference: Reference, ) -> Result { tracing::info!(name:"rspack_version", version = rspack_version!()); @@ -168,10 +170,36 @@ impl JsCompiler { bp.append_to(env, &mut this, &mut plugins)?; } + let use_input_fs = options.experiments.use_input_file_system.take(); let compiler_options: rspack_core::CompilerOptions = options.try_into().to_napi_result()?; tracing::debug!(name:"normalized_options", options=?&compiler_options); + let input_file_system: Option> = input_filesystem.and_then(|fs| { + use_input_fs.and_then(|use_input_file_system| { + let node_fs = NodeFileSystem::new(fs).expect("Failed to create readable filesystem"); + + match use_input_file_system { + WithFalse::False => None, + WithFalse::True(allowlist) => { + if allowlist.is_empty() { + return None; + } + let binding: Arc = Arc::new(HybridFileSystem::new( + allowlist, + node_fs, + NativeFileSystem::new(compiler_options.resolve.pnp.unwrap_or(false)), + )); + Some(binding) + } + } + }) + }); + + if let Some(fs) = &input_file_system { + resolver_factory_reference.input_filesystem = fs.clone(); + } + let resolver_factory = (*resolver_factory_reference).get_resolver_factory(compiler_options.resolve.clone()); let loader_resolver_factory = (*resolver_factory_reference) @@ -198,7 +226,7 @@ impl JsCompiler { .to_napi_result_with_message(|e| format!("Failed to create writable filesystem: {e}"))?, )), intermediate_filesystem, - None, + input_file_system, Some(resolver_factory), Some(loader_resolver_factory), ); diff --git a/crates/node_binding/src/raw_options/raw_experiments/mod.rs b/crates/node_binding/src/raw_options/raw_experiments/mod.rs index 6b99e38611ff..08838b576ebf 100644 --- a/crates/node_binding/src/raw_options/raw_experiments/mod.rs +++ b/crates/node_binding/src/raw_options/raw_experiments/mod.rs @@ -7,6 +7,7 @@ use raw_cache::{normalize_raw_experiment_cache_options, RawExperimentCacheOption use raw_incremental::RawIncremental; use raw_rspack_future::RawRspackFuture; use rspack_core::{incremental::IncrementalOptions, Experiments}; +use rspack_regex::RspackRegex; use super::WithFalse; @@ -23,6 +24,8 @@ pub struct RawExperiments { ts_type = r#"boolean | { type: "persistent" } & RawExperimentCacheOptionsPersistent | { type: "memory" }"# )] pub cache: RawExperimentCacheOptions, + #[napi(ts_type = "false | Array")] + pub use_input_file_system: Option>>, } impl From for Experiments { diff --git a/crates/node_binding/src/resolver.rs b/crates/node_binding/src/resolver.rs index d2d6fbe1fd3e..8bfab897fc34 100644 --- a/crates/node_binding/src/resolver.rs +++ b/crates/node_binding/src/resolver.rs @@ -44,15 +44,21 @@ impl JsResolver { path: String, request: String, ) -> napi::Result> { - block_on(async move { - match self.resolver.resolve(Path::new(&path), &request).await { - Ok(rspack_core::ResolveResult::Resource(resource)) => { - Ok(Either::A(ResourceData::from(resource).into())) - } - Ok(rspack_core::ResolveResult::Ignored) => Ok(Either::B(false)), - Err(err) => Err(napi::Error::from_reason(format!("{err:?}"))), + block_on(self._resolve(path, request)) + } + + async fn _resolve( + &self, + path: String, + request: String, + ) -> napi::Result> { + match self.resolver.resolve(Path::new(&path), &request).await { + Ok(rspack_core::ResolveResult::Resource(resource)) => { + Ok(Either::A(ResourceData::from(resource).into())) } - }) + Ok(rspack_core::ResolveResult::Ignored) => Ok(Either::B(false)), + Err(err) => Err(napi::Error::from_reason(format!("{err:?}"))), + } } #[napi( diff --git a/packages/rspack-test-tools/src/helper/util/checkStats.js b/packages/rspack-test-tools/src/helper/util/checkStats.js index 555df631584b..996e4cbaaa61 100644 --- a/packages/rspack-test-tools/src/helper/util/checkStats.js +++ b/packages/rspack-test-tools/src/helper/util/checkStats.js @@ -11,7 +11,7 @@ exports.checkChunkModules = function checkChunkModules( const chunkModules = chunk.modules.map(m => m.identifier); if (strict && expectedModules.length !== chunkModules.length) { throw new Error( - `expect chunk ${chunkId} has ${chunkModules.length} modules: ${chunkModules}\nbut received ${chunkModules.length} modules` + `expect chunk ${chunkId} has ${expectedModules.length} modules: ${expectedModules}\nbut received ${chunkModules.length} modules` ); } diff --git a/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap b/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap index 90ebaf141216..d5d7cbcd62ec 100644 --- a/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap +++ b/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap @@ -52,6 +52,7 @@ Object { }, }, topLevelAwait: true, + useInputFileSystem: false, }, externals: undefined, externalsPresets: Object { diff --git a/packages/rspack-test-tools/tests/configCases/input-file-system/disk.js b/packages/rspack-test-tools/tests/configCases/input-file-system/disk.js new file mode 100644 index 000000000000..e3f45c7d81d6 --- /dev/null +++ b/packages/rspack-test-tools/tests/configCases/input-file-system/disk.js @@ -0,0 +1,3 @@ +it("should load a file from the disk", ()=>{ + expect(42).toBe(42); +}) \ No newline at end of file diff --git a/packages/rspack-test-tools/tests/configCases/input-file-system/webpack.config.js b/packages/rspack-test-tools/tests/configCases/input-file-system/webpack.config.js new file mode 100644 index 000000000000..083f0ebcb4e9 --- /dev/null +++ b/packages/rspack-test-tools/tests/configCases/input-file-system/webpack.config.js @@ -0,0 +1,59 @@ +const assert = require("node:assert"); + +let readFileCalled = false; +let stateCalled = false; + +/** @type {import("../../../../").Configuration} */ +module.exports = { + entry: { + index: "./virtual_index.js" + }, + plugins: [ + { + apply: compiler => { + compiler.hooks.afterCompile.tap("SimpleInputFileSystem", () => { + assert(readFileCalled, "readFile should be called"); + assert(stateCalled, "stat should be called"); + }); + compiler.hooks.beforeCompile.tap("SimpleInputFileSystem", () => { + // simple file system just works for test case + compiler.inputFileSystem = { + readFile(p, cb) { + readFileCalled = true; + cb( + null, + ` + require("./disk.js"); + it("should read file simple file",()=>{ + expect(1).toBe(1); + })` + ); + }, + stat(p, cb) { + stateCalled = true; + cb(null, { + isFile() { + return true; + }, + isDirectory() { + return false; + }, + isSymbolicLink() { + return false; + }, + atimeMs: 1749025843289.6816, + mtimeMs: 1749025842638.571, + ctimeMs: 1749025842638.571, + birthtimeMs: 1749025385767.096, + size: 1000 + }); + } + }; + }); + } + } + ], + experiments: { + useInputFileSystem: [/virtual_.*\.js/] + } +}; diff --git a/packages/rspack-test-tools/tests/configCases/module/build-info/rspack.config.js b/packages/rspack-test-tools/tests/configCases/module/build-info/rspack.config.js index ec86c06e56cc..4dc922c76aff 100644 --- a/packages/rspack-test-tools/tests/configCases/module/build-info/rspack.config.js +++ b/packages/rspack-test-tools/tests/configCases/module/build-info/rspack.config.js @@ -12,10 +12,16 @@ class Plugin { expect(Object.keys(entryModule.buildInfo.assets)).toContain("foo.txt"); expect(entryModule.buildInfo.fileDependencies.size).toBe(1); - expect(entryModule.buildInfo.fileDependencies.has(path.join(__dirname, "index.js"))).toBe(true); + expect( + entryModule.buildInfo.fileDependencies.has( + path.join(__dirname, "index.js") + ) + ).toBe(true); expect(entryModule.buildInfo.buildDependencies.size).toBe(1); - expect(entryModule.buildInfo.buildDependencies.has("./build.txt")).toBe(true); + expect(entryModule.buildInfo.buildDependencies.has("./build.txt")).toBe( + true + ); expect(entryModule.buildInfo.contextDependencies.size).toBe(0); diff --git a/packages/rspack-test-tools/tests/configCases/resolver/resolve/rspack.config.js b/packages/rspack-test-tools/tests/configCases/resolver/resolve/rspack.config.js index ff55498c9937..de7f7073dc53 100644 --- a/packages/rspack-test-tools/tests/configCases/resolver/resolve/rspack.config.js +++ b/packages/rspack-test-tools/tests/configCases/resolver/resolve/rspack.config.js @@ -1,28 +1,35 @@ const path = require("path"); class Plugin { - /** - * @param {import("@rspack/core").Compiler} compiler - */ - apply(compiler) { - compiler.hooks.compilation.tap("PLUGIN", (compilation) => { - compilation.hooks.finishModules.tapAsync("PLUGIN", (modules, callback) => { - const normalResolver = compiler.resolverFactory.get("normal"); - normalResolver.resolve({}, __dirname, "./index.js", {}, (error, res, req) => { - expect(error).toBeNull(); - expect(res).toBe(path.join(__dirname, "/index.js")); - expect(req.resource).toBe(path.join(__dirname, "/index.js")); - callback(); - }); - }); - }); - } + /** + * @param {import("@rspack/core").Compiler} compiler + */ + apply(compiler) { + compiler.hooks.compilation.tap("PLUGIN", compilation => { + compilation.hooks.finishModules.tapAsync( + "PLUGIN", + (modules, callback) => { + const normalResolver = compiler.resolverFactory.get("normal"); + normalResolver.resolve( + {}, + __dirname, + "./index.js", + {}, + (error, res, req) => { + expect(error).toBeNull(); + expect(res).toBe(path.join(__dirname, "/index.js")); + expect(req.resource).toBe(path.join(__dirname, "/index.js")); + callback(); + } + ); + } + ); + }); + } } /** @type {import("@rspack/core").Configuration} */ module.exports = { - entry: "./index.js", - plugins: [ - new Plugin() - ] + entry: "./index.js", + plugins: [new Plugin()] }; diff --git a/packages/rspack/etc/core.api.md b/packages/rspack/etc/core.api.md index cf3fbc5cf613..4c3adfe0fa02 100644 --- a/packages/rspack/etc/core.api.md +++ b/packages/rspack/etc/core.api.md @@ -2458,6 +2458,7 @@ export type Experiments = { rspackFuture?: RspackFutureOptions; buildHttp?: HttpUriOptions; parallelLoader?: boolean; + useInputFileSystem?: UseInputFileSystem; }; // @public (undocumented) @@ -2515,6 +2516,8 @@ export interface ExperimentsNormalized { rspackFuture?: RspackFutureOptions; // (undocumented) topLevelAwait?: boolean; + // (undocumented) + useInputFileSystem?: false | RegExp[]; } // @public (undocumented) @@ -6585,6 +6588,7 @@ declare namespace rspackExports { Incremental, IncrementalPresets, HttpUriOptions, + UseInputFileSystem, Experiments, Watch, WatchOptions, @@ -8845,6 +8849,9 @@ type UpdateOperator = "++" | "--"; // @public type UsageStateType = 0 | 1 | 2 | 3 | 4; +// @public +export type UseInputFileSystem = false | RegExp[]; + // @public (undocumented) export const util: { createHash: (algorithm: "debug" | "xxhash64" | "md4" | "native-md4" | (string & {}) | (new () => default_2)) => default_2; diff --git a/packages/rspack/module.d.ts b/packages/rspack/module.d.ts index ef3309afc59f..15303eed397b 100644 --- a/packages/rspack/module.d.ts +++ b/packages/rspack/module.d.ts @@ -211,7 +211,7 @@ declare namespace Rspack { interface Process { env: { [key: string]: any; - NODE_ENV: 'development' | 'production' | (string & {}); + NODE_ENV: "development" | "production" | (string & {}); }; } } diff --git a/packages/rspack/src/Compiler.ts b/packages/rspack/src/Compiler.ts index 5d4be20b47d9..35dd28e2d03e 100644 --- a/packages/rspack/src/Compiler.ts +++ b/packages/rspack/src/Compiler.ts @@ -19,6 +19,7 @@ import type { Chunk } from "./Chunk"; import { Compilation } from "./Compilation"; import { ContextModuleFactory } from "./ContextModuleFactory"; import { + ThreadsafeInputNodeFS, ThreadsafeIntermediateNodeFS, ThreadsafeOutputNodeFS } from "./FileSystem"; @@ -836,6 +837,12 @@ class Compiler { this.#registers = this.#createHooksRegisters(); + const inputFileSystem = + this.inputFileSystem && + ThreadsafeInputNodeFS.needsBinding(options.experiments.useInputFileSystem) + ? ThreadsafeInputNodeFS.__to_binding(this.inputFileSystem) + : undefined; + this.#instance = new instanceBinding.JsCompiler( this.compilerPath, rawOptions, @@ -845,6 +852,7 @@ class Compiler { this.intermediateFileSystem ? ThreadsafeIntermediateNodeFS.__to_binding(this.intermediateFileSystem) : undefined, + inputFileSystem, ResolverFactory.__to_binding(this.resolverFactory) ); diff --git a/packages/rspack/src/FileSystem.ts b/packages/rspack/src/FileSystem.ts index 49f543bb9e46..3dbe5d0f18ce 100644 --- a/packages/rspack/src/FileSystem.ts +++ b/packages/rspack/src/FileSystem.ts @@ -3,6 +3,7 @@ import type { NodeFsStats, ThreadsafeNodeFS } from "@rspack/binding"; import { type IStats, + type InputFileSystem, type IntermediateFileSystem, type OutputFileSystem, mkdirp, @@ -24,6 +25,7 @@ const NOOP_FILESYSTEM: ThreadsafeNodeFS = { readFile: ASYNC_NOOP, stat: ASYNC_NOOP, lstat: ASYNC_NOOP, + realpath: ASYNC_NOOP, open: ASYNC_NOOP, rename: ASYNC_NOOP, close: ASYNC_NOOP, @@ -34,6 +36,124 @@ const NOOP_FILESYSTEM: ThreadsafeNodeFS = { readToEnd: ASYNC_NOOP }; +function __to_binding_stat(stat: IStats): NodeFsStats { + return { + isFile: stat.isFile(), + isDirectory: stat.isDirectory(), + isSymlink: stat.isSymbolicLink(), + atimeMs: stat.atimeMs ?? toMs(stat.atime), + mtimeMs: stat.mtimeMs ?? toMs(stat.mtime), + ctimeMs: stat.ctimeMs ?? toMs(stat.ctime), + birthtimeMs: stat.birthtimeMs ?? toMs(stat.birthtime), + size: stat.size + }; +} + +function toMs(i: Date | number): number { + if ((i as Date).getTime) { + return (i as Date).getTime(); + } + return i as number; +} + +class ThreadsafeInputNodeFS implements ThreadsafeNodeFS { + writeFile!: (name: string, content: Buffer) => Promise; + removeFile!: (name: string) => Promise; + mkdir!: (name: string) => Promise; + mkdirp!: (name: string) => Promise; + removeDirAll!: (name: string) => Promise; + readDir!: (name: string) => Promise; + readFile!: (name: string) => Promise; + stat!: (name: string) => Promise; + lstat!: (name: string) => Promise; + realpath!: (name: string) => Promise; + open!: (name: string, flags: string) => Promise; + rename!: (from: string, to: string) => Promise; + close!: (fd: number) => Promise; + write!: ( + fd: number, + content: Buffer, + position: number + ) => Promise; + writeAll!: (fd: number, content: Buffer) => Promise; + read!: ( + fd: number, + length: number, + position: number + ) => Promise; + readUntil!: ( + fd: number, + code: number, + position: number + ) => Promise; + readToEnd!: (fd: number, position: number) => Promise; + + constructor(fs?: InputFileSystem) { + Object.assign(this, NOOP_FILESYSTEM); + if (!fs) { + return; + } + // On the rust side, ReadableFileSystem only uses the readFile and stats + // TODO: is `memoizeFn` necessary? + this.readDir = memoizeFn(() => { + const readDirFn = util.promisify(fs.readdir.bind(fs)); + return async (filePath: string) => { + const res = await readDirFn(filePath); + return res as string[]; + }; + }); + this.readFile = memoizeFn(() => util.promisify(fs.readFile.bind(fs))); + this.stat = memoizeFn(() => { + return (name: string) => { + return new Promise((resolve, reject) => { + fs.stat(name, (err, stats) => { + if (err) { + return reject(err); + } + resolve(stats && __to_binding_stat(stats)); + }); + }); + }; + }); + this.lstat = memoizeFn(() => { + return (name: string) => { + return new Promise((resolve, reject) => { + (fs.lstat || fs.stat)(name, (err, stats) => { + if (err) { + return reject(err); + } + resolve(stats && __to_binding_stat(stats)); + }); + }); + }; + }); + this.realpath = memoizeFn(() => { + return (name: string) => { + return new Promise((resolve, reject) => { + if (fs.realpath) { + fs.realpath(name, (err, path) => { + if (err) { + return reject(err); + } + resolve(path); + }); + } else { + reject(new Error("fs.realpath is not a function")); + } + }); + }; + }); + } + + static __to_binding(fs?: InputFileSystem) { + return new this(fs); + } + + static needsBinding(ifs?: false | RegExp[]) { + return Array.isArray(ifs) && ifs.length > 0; + } +} + class ThreadsafeOutputNodeFS implements ThreadsafeNodeFS { writeFile!: (name: string, content: Buffer) => Promise; removeFile!: (name: string) => Promise; @@ -44,6 +164,7 @@ class ThreadsafeOutputNodeFS implements ThreadsafeNodeFS { readFile!: (name: string) => Promise; stat!: (name: string) => Promise; lstat!: (name: string) => Promise; + realpath!: (name: string) => Promise; open!: (name: string, flags: string) => Promise; rename!: (from: string, to: string) => Promise; close!: (fd: number) => Promise; @@ -87,14 +208,14 @@ class ThreadsafeOutputNodeFS implements ThreadsafeNodeFS { const statFn = util.promisify(fs.stat.bind(fs)); return async (filePath: string) => { const res = await statFn(filePath); - return res && ThreadsafeOutputNodeFS.__to_binding_stat(res); + return res && __to_binding_stat(res); }; }); this.lstat = memoizeFn(() => { const statFn = util.promisify((fs.lstat || fs.stat).bind(fs)); return async (filePath: string) => { const res = await statFn(filePath); - return res && ThreadsafeOutputNodeFS.__to_binding_stat(res); + return res && __to_binding_stat(res); }; }); } @@ -102,19 +223,6 @@ class ThreadsafeOutputNodeFS implements ThreadsafeNodeFS { static __to_binding(fs?: OutputFileSystem) { return new this(fs); } - - static __to_binding_stat(stat: IStats): NodeFsStats { - return { - isFile: stat.isFile(), - isDirectory: stat.isDirectory(), - isSymlink: stat.isSymbolicLink(), - atimeMs: stat.atimeMs, - mtimeMs: stat.atimeMs, - ctimeMs: stat.atimeMs, - birthtimeMs: stat.birthtimeMs, - size: stat.size - }; - } } class ThreadsafeIntermediateNodeFS extends ThreadsafeOutputNodeFS { @@ -203,4 +311,8 @@ class ThreadsafeIntermediateNodeFS extends ThreadsafeOutputNodeFS { } } -export { ThreadsafeOutputNodeFS, ThreadsafeIntermediateNodeFS }; +export { + ThreadsafeInputNodeFS, + ThreadsafeOutputNodeFS, + ThreadsafeIntermediateNodeFS +}; diff --git a/packages/rspack/src/config/defaults.ts b/packages/rspack/src/config/defaults.ts index 84a23a1602b4..bb2b304fde0e 100644 --- a/packages/rspack/src/config/defaults.ts +++ b/packages/rspack/src/config/defaults.ts @@ -253,6 +253,10 @@ const applyExperimentsDefaults = ( // IGNORE(experiments.parallelLoader): Rspack specific configuration for parallel loader execution D(experiments, "parallelLoader", false); + + // IGNORE(experiments.useInputFileSystem): Rspack specific configuration + // Enable `useInputFileSystem` will introduce much more fs overheads, So disable by default. + D(experiments, "useInputFileSystem", false); }; const applybundlerInfoDefaults = ( diff --git a/packages/rspack/src/config/normalization.ts b/packages/rspack/src/config/normalization.ts index dd4ea70bbeeb..54d0ad73477f 100644 --- a/packages/rspack/src/config/normalization.ts +++ b/packages/rspack/src/config/normalization.ts @@ -354,7 +354,8 @@ export const getNormalizedRspackOptions = ( ), parallelCodeSplitting: experiments.parallelCodeSplitting, buildHttp: experiments.buildHttp, - parallelLoader: experiments.parallelLoader + parallelLoader: experiments.parallelLoader, + useInputFileSystem: experiments.useInputFileSystem })), watch: config.watch, watchOptions: cloneObject(config.watchOptions), @@ -628,6 +629,7 @@ export interface ExperimentsNormalized { rspackFuture?: RspackFutureOptions; buildHttp?: HttpUriPluginOptions; parallelLoader?: boolean; + useInputFileSystem?: false | RegExp[]; } export type IgnoreWarningsNormalized = (( diff --git a/packages/rspack/src/config/types.ts b/packages/rspack/src/config/types.ts index 61be58bad221..6083f7b54769 100644 --- a/packages/rspack/src/config/types.ts +++ b/packages/rspack/src/config/types.ts @@ -2628,6 +2628,11 @@ export type IncrementalPresets = */ export type HttpUriOptions = HttpUriPluginOptions; +/** + * Options for experiments.useInputFileSystem + */ +export type UseInputFileSystem = false | RegExp[]; + /** * Experimental features configuration. */ @@ -2702,6 +2707,11 @@ export type Experiments = { * @default false */ parallelLoader?: boolean; + /** + * Enable Node.js input file system + * @default false + */ + useInputFileSystem?: UseInputFileSystem; }; //#endregion diff --git a/packages/rspack/src/config/zod.ts b/packages/rspack/src/config/zod.ts index 20801ceea5fe..bb83a92841fa 100644 --- a/packages/rspack/src/config/zod.ts +++ b/packages/rspack/src/config/zod.ts @@ -1493,6 +1493,11 @@ const buildHttpOptions = z.object({ .optional() }) satisfies z.ZodType; +const useInputFileSystem = z.union([ + z.literal(false), + z.array(z.instanceof(RegExp)) +]) satisfies z.ZodType; + const experiments = z.strictObject({ cache: z.boolean().optional().or(experimentCacheOptions), lazyCompilation: z.boolean().optional().or(lazyCompilationOptions), @@ -1512,7 +1517,8 @@ const experiments = z.strictObject({ futureDefaults: z.boolean().optional(), rspackFuture: rspackFutureOptions.optional(), buildHttp: buildHttpOptions.optional(), - parallelLoader: z.boolean().optional() + parallelLoader: z.boolean().optional(), + useInputFileSystem: useInputFileSystem.optional() }) satisfies z.ZodType; //#endregion diff --git a/tests/webpack-test/Defaults.unittest.js b/tests/webpack-test/Defaults.unittest.js index 9b04a340a7f9..223d455d3a98 100644 --- a/tests/webpack-test/Defaults.unittest.js +++ b/tests/webpack-test/Defaults.unittest.js @@ -103,6 +103,7 @@ describe("snapshots", () => { "outputModule": false, "syncWebAssembly": false, "topLevelAwait": true, + "useInputFileSystem": false, }, "externals": undefined, "externalsPresets": Object { diff --git a/website/docs/en/config/experiments.mdx b/website/docs/en/config/experiments.mdx index 5ded1da77167..88ce9e6221de 100644 --- a/website/docs/en/config/experiments.mdx +++ b/website/docs/en/config/experiments.mdx @@ -828,3 +828,77 @@ import { something } from 'https://example.com/module.js'; // Or import assets import imageUrl from 'https://example.com/image.png'; ``` + +## experiments.useInputFileSystem + + + +- **Type:** `false | RegExp[]` +- **Default:** `false` + +By default, Rspack reads files from disk using a native file system. +However, it is possible to change the file system using a different kind of file system. +To accomplish this, one can change the inputFileSystem. For example, you can replace the default inputFileSystem with memfs to virtual Modules. + +But due to the overheads calling file system implemented in Node.js side, it will slow down Rspack a lot. +So we make a trade off by providing the `useInputFileSystem` config, to tell rspack to read file from the native file system or from modified inputFileSystem. + +In below example, you can simply replace the default input file system to any file system satisfied the [`InputFileSystem`](/api/javascript-api/compiler#inputfilesystem-1) interface. + +:::info Note +The replacing of `compiler.inputFileSystem` will only take effect before `compiler.run` called; Replacing after `compiler.run` will not take effect. +::: + +More detailed case can be found [here](https://github.com/web-infra-dev/rspack/tree/main/packages/rspack-test-tools/tests/configCases/input-file-system/webpack.config.js) + +```js title="rspack.config.mjs" +export default { + entry: { + index: './virtual_index.js', + }, + plugins: [ + { + apply: compiler => { + compiler.hooks.beforeCompile.tap('SimpleInputFileSystem', () => { + compiler.inputFileSystem = { + readFile(path, cb) { + cb(null, `// the file content`); + }, + stat(p, cb) { + cb(null, fsState); + }, + }; + }); + }, + }, + ], + experiments: { + useInputFileSystem: [/virtual_.*\.js/], + }, +}; +``` + +### Work with webpack-virtual-modules + +```js title="rspack.config.mjs" +import VirtualModulesPlugin from 'webpack-virtual-modules'; + +var virtualModules = new VirtualModulesPlugin({ + 'virtual_entry.js': ` + require("./src/another_virtual.js"); + require("./src/disk_file.js") + `, + 'src/another_virtual.js': 'module.exports = 42', +}); + +export default { + entry: './virtual_entry.js', + plugins: [virtualModules], + experiments: { + useInputFileSystem: [/.*virtual.*\.js$/], + }, +}; +``` + +When access to `virtual_entry.js` and `src/another_virtual.js` which match the regular expressions of `experiments.useInputFileSystem`, +Rspack will use the input file system wrapped by `VirtualModulesPlugin`; other than that, `src/disk_file.js` will be accessed by the native file system. diff --git a/website/docs/zh/config/experiments.mdx b/website/docs/zh/config/experiments.mdx index 58d7c3c35d5f..ab3f041a9d41 100644 --- a/website/docs/zh/config/experiments.mdx +++ b/website/docs/zh/config/experiments.mdx @@ -829,3 +829,77 @@ import { something } from 'https://example.com/module.js'; // Or import assets import imageUrl from 'https://example.com/image.png'; ``` + +## experiments.useInputFileSystem + + + +- **类型:** `false | RegExp[]` +- **默认值:** `false` + +默认情况下，Rspack 使用原生文件系统从磁盘读取文件。 +但你也可以通过更换 inputFileSystem，使用其他类型的文件系统。例如，可以用 memfs 替代默认文件系统来支持虚拟模块。 + +不过，由于调用 Node.js 实现的文件系统存在性能开销，会显著拖慢 Rspack 的运行速度。 +因此，Rspack 提供了 `useInputFileSystem` 配置项，用于控制是使用原生文件系统还是自定义的 inputFileSystem。 +一般项目中，只有少量文件需要通过自定义文件系统访问，合理配置 `experiments.useInputFileSystem` 可以确保 Rspack 高效运行。 + +下面的例子演示了如何替换默认的 inputFileSystem 为一个自定义的文件系统，只要自定义文件系统满足 [`InputFileSystem`](/zh/api/javascript-api/compiler#inputfilesystem-1) 的接口定义。 +更多细节参考[测试用例](https://github.com/web-infra-dev/rspack/tree/main/packages/rspack-test-tools/tests/configCases/input-file-system/webpack.config.js)。 + +```js title="rspack.config.mjs" +export default { + entry: { + index: './virtual_index.js', + }, + plugins: [ + { + apply: compiler => { + compiler.hooks.beforeCompile.tap('SimpleInputFileSystem', () => { + compiler.inputFileSystem = { + readFile(path, cb) { + cb(null, `// the file content`); + }, + stat(p, cb) { + cb(null, fsState); + }, + }; + }); + }, + }, + ], + experiments: { + useInputFileSystem: [/virtual_.*\.js/], + }, +}; +``` + +:::info 注意 +只有在 `compiler.run` 执行前替换的 `compiler.inputFileSystem` 才会生效；在 `compiler.run` 执行后替换将不会生效。 +::: + +### 配合 webpack-virtual-modules 使用 + +例子： + +```js title="rspack.config.mjs" +import VirtualModulesPlugin from 'webpack-virtual-modules'; + +var virtualModules = new VirtualModulesPlugin({ + 'virtual_entry.js': ` + require("./src/another_virtual.js"); + require("./src/disk_file.js") + `, + 'src/another_virtual.js': 'module.exports = 42', +}); + +export default { + entry: './virtual_entry.js', + plugins: [virtualModules], + experiments: { + useInputFileSystem: [/.*virtual.*\.js$/], + }, +}; +``` + +当访问 `virtual_entry.js` 和 `src/another_virtual.js` 时，它们匹配了 `experiments.useInputFileSystem` 中的正则规则，Rspack 会通过 `VirtualModulesPlugin` 包装的输入文件系统读取它们；而像 `src/disk_file.js`，则会使用原生文件系统读取。