From d90be1f2661197ff7a8cbf9ff58fbf7811d8001e Mon Sep 17 00:00:00 2001 From: Alexander Lyon Date: Tue, 5 Nov 2024 05:14:08 +0000 Subject: [PATCH] thread tracing through to resolve results --- crates/next-core/src/next_build.rs | 5 +- .../src/next_dynamic/dynamic_transition.rs | 5 +- crates/next-core/src/next_import_map.rs | 69 +++++++-- crates/next-core/src/next_server/resolve.rs | 34 +++-- crates/next-core/src/next_shared/resolve.rs | 28 ++-- .../src/next_shared/webpack_rules/mod.rs | 15 +- .../next/src/build/swc/generated-native.d.ts | 14 -- .../turbopack-core/src/chunk/chunk_group.rs | 27 +++- .../crates/turbopack-core/src/chunk/mod.rs | 51 ++++++- .../crates/turbopack-core/src/context.rs | 39 ++++- .../turbopack-core/src/introspect/utils.rs | 6 + .../crates/turbopack-core/src/issue/mod.rs | 1 + .../crates/turbopack-core/src/issue/module.rs | 35 +++++ .../turbopack-core/src/issue/resolve.rs | 22 +-- turbopack/crates/turbopack-core/src/lib.rs | 1 + .../mod.rs => turbopack-core/src/rebase.rs} | 3 +- .../turbopack-core/src/reference/mod.rs | 40 +++++ .../crates/turbopack-core/src/resolve/mod.rs | 137 ++++++++++++++++-- .../turbopack-core/src/resolve/options.rs | 62 +++++--- .../turbopack-core/src/resolve/pattern.rs | 18 ++- .../turbopack-core/src/virtual_output.rs | 33 ++++- .../crates/turbopack-css/src/module_asset.rs | 12 +- .../src/references/esm/base.rs | 4 +- .../src/references/external_module.rs | 31 +++- .../src/references/pattern_mapping.rs | 2 +- .../turbopack-node/src/transforms/postcss.rs | 10 +- .../crates/turbopack-resolve/src/resolve.rs | 35 ++++- .../turbopack-resolve/src/typescript.rs | 10 +- .../crates/turbopack-tests/tests/execution.rs | 10 +- .../turbopack/benches/node_file_trace.rs | 2 +- .../crates/turbopack/examples/turbopack.rs | 3 +- turbopack/crates/turbopack/src/lib.rs | 21 ++- .../crates/turbopack/src/transition/mod.rs | 9 +- .../crates/turbopack/tests/node-file-trace.rs | 2 +- 34 files changed, 633 insertions(+), 163 deletions(-) create mode 100644 turbopack/crates/turbopack-core/src/issue/module.rs rename turbopack/crates/{turbopack/src/rebase/mod.rs => turbopack-core/src/rebase.rs} (98%) diff --git a/crates/next-core/src/next_build.rs b/crates/next-core/src/next_build.rs index 56267568759597..fb3028390bea2e 100644 --- a/crates/next-core/src/next_build.rs +++ b/crates/next-core/src/next_build.rs @@ -1,7 +1,7 @@ use anyhow::Result; use turbo_tasks::{RcStr, ResolvedVc, Vc}; use turbo_tasks_fs::FileSystemPath; -use turbopack_core::resolve::{options::ImportMapping, ExternalType}; +use turbopack_core::resolve::{options::ImportMapping, ExternalTraced, ExternalType}; use crate::next_import_map::get_next_package; @@ -23,11 +23,14 @@ pub async fn get_postcss_package_mapping( #[turbo_tasks::function] pub async fn get_external_next_compiled_package_mapping( + project_path: ResolvedVc, package_name: Vc, ) -> Result> { Ok(ImportMapping::Alternatives(vec![ImportMapping::External( Some(format!("next/dist/compiled/{}", &*package_name.await?).into()), ExternalType::CommonJs, + ExternalTraced::Traced(project_path), + None, ) .resolved_cell()]) .cell()) diff --git a/crates/next-core/src/next_dynamic/dynamic_transition.rs b/crates/next-core/src/next_dynamic/dynamic_transition.rs index 8e958e0bd61f5a..097adb09248e2c 100644 --- a/crates/next-core/src/next_dynamic/dynamic_transition.rs +++ b/crates/next-core/src/next_dynamic/dynamic_transition.rs @@ -48,14 +48,15 @@ impl Transition for NextDynamicTransition { module_asset_context, Value::new(ReferenceType::Undefined), ) + .try_into_module() .await? { - ProcessResult::Module(client_module) => ProcessResult::Module(ResolvedVc::upcast( + Some(client_module) => ProcessResult::Module(ResolvedVc::upcast( NextDynamicEntryModule::new(*client_module) .to_resolved() .await?, )), - ProcessResult::Ignore => ProcessResult::Ignore, + None => ProcessResult::Ignore, } .cell()) } diff --git a/crates/next-core/src/next_import_map.rs b/crates/next-core/src/next_import_map.rs index df1c325eaca4d8..d86212f7f9f224 100644 --- a/crates/next-core/src/next_import_map.rs +++ b/crates/next-core/src/next_import_map.rs @@ -10,7 +10,7 @@ use turbopack_core::{ options::{ConditionValue, ImportMap, ImportMapping, ResolvedMap}, parse::Request, pattern::Pattern, - resolve, AliasPattern, ExternalType, ResolveAliasMap, SubpathValue, + resolve, AliasPattern, ExternalTraced, ExternalType, ResolveAliasMap, SubpathValue, }, source::Source, }; @@ -239,7 +239,7 @@ pub async fn get_next_client_import_map( /// Computes the Next-specific client import map. #[turbo_tasks::function] -pub async fn get_next_build_import_map() -> Result> { +pub async fn get_next_build_import_map(project_path: ResolvedVc) -> Result> { let mut import_map = ImportMap::empty(); insert_package_alias( @@ -248,15 +248,26 @@ pub async fn get_next_build_import_map() -> Result> { next_js_fs().root().to_resolved().await?, ); - let external = ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(); + let external = ImportMapping::External( + None, + ExternalType::CommonJs, + ExternalTraced::Traced(project_path), + None, + ) + .resolved_cell(); import_map.insert_exact_alias("next", external); import_map.insert_wildcard_alias("next/", external); import_map.insert_exact_alias("styled-jsx", external); import_map.insert_exact_alias( "styled-jsx/style", - ImportMapping::External(Some("styled-jsx/style.js".into()), ExternalType::CommonJs) - .resolved_cell(), + ImportMapping::External( + Some("styled-jsx/style.js".into()), + ExternalType::CommonJs, + ExternalTraced::Traced(project_path), + None, + ) + .resolved_cell(), ); import_map.insert_wildcard_alias("styled-jsx/", external); @@ -321,7 +332,14 @@ pub async fn get_next_server_import_map( let ty = ty.into_value(); - let external = ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(); + let external = ImportMapping::External( + None, + ExternalType::CommonJs, + // TODO(arlyon): wiring up in a follow up PR + ExternalTraced::Untraced, + None, + ) + .resolved_cell(); import_map.insert_exact_alias("next/dist/server/require-hook", external); match ty { @@ -336,8 +354,13 @@ pub async fn get_next_server_import_map( import_map.insert_exact_alias("styled-jsx", external); import_map.insert_exact_alias( "styled-jsx/style", - ImportMapping::External(Some("styled-jsx/style.js".into()), ExternalType::CommonJs) - .resolved_cell(), + ImportMapping::External( + Some("styled-jsx/style.js".into()), + ExternalType::CommonJs, + ExternalTraced::Traced(project_path), + None, + ) + .resolved_cell(), ); import_map.insert_wildcard_alias("styled-jsx/", external); // TODO: we should not bundle next/dist/build/utils in the pages renderer at all @@ -562,12 +585,12 @@ async fn insert_next_server_special_aliases( let external_cjs_if_node = move |context_dir: ResolvedVc, request: &str| match runtime { NextRuntime::Edge => request_to_import_mapping(context_dir, request), - NextRuntime::NodeJs => external_request_to_cjs_import_mapping(request), + NextRuntime::NodeJs => external_request_to_cjs_import_mapping(context_dir, request), }; let external_esm_if_node = move |context_dir: ResolvedVc, request: &str| match runtime { NextRuntime::Edge => request_to_import_mapping(context_dir, request), - NextRuntime::NodeJs => external_request_to_esm_import_mapping(request), + NextRuntime::NodeJs => external_request_to_esm_import_mapping(context_dir, request), }; import_map.insert_exact_alias( @@ -1119,12 +1142,30 @@ fn request_to_import_mapping( /// Creates a direct import mapping to the result of resolving an external /// request. -fn external_request_to_cjs_import_mapping(request: &str) -> ResolvedVc { - ImportMapping::External(Some(request.into()), ExternalType::CommonJs).resolved_cell() +fn external_request_to_cjs_import_mapping( + context_dir: ResolvedVc, + request: &str, +) -> ResolvedVc { + ImportMapping::External( + Some(request.into()), + ExternalType::CommonJs, + ExternalTraced::Traced(context_dir), + Some(context_dir), + ) + .resolved_cell() } /// Creates a direct import mapping to the result of resolving an external /// request. -fn external_request_to_esm_import_mapping(request: &str) -> ResolvedVc { - ImportMapping::External(Some(request.into()), ExternalType::EcmaScriptModule).resolved_cell() +fn external_request_to_esm_import_mapping( + context_dir: ResolvedVc, + request: &str, +) -> ResolvedVc { + ImportMapping::External( + Some(request.into()), + ExternalType::EcmaScriptModule, + ExternalTraced::Traced(context_dir), + Some(context_dir), + ) + .resolved_cell() } diff --git a/crates/next-core/src/next_server/resolve.rs b/crates/next-core/src/next_server/resolve.rs index c5e3a5baabd462..551a9578f89635 100644 --- a/crates/next-core/src/next_server/resolve.rs +++ b/crates/next-core/src/next_server/resolve.rs @@ -12,8 +12,8 @@ use turbopack_core::{ parse::Request, pattern::Pattern, plugin::{AfterResolvePlugin, AfterResolvePluginCondition}, - resolve, ExternalType, FindContextFileResult, ResolveResult, ResolveResultItem, - ResolveResultOption, + resolve, ExternalTraced, ExternalType, FindContextFileResult, ResolveResult, + ResolveResultItem, ResolveResultOption, }, source::Source, }; @@ -375,10 +375,12 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { (FileType::CommonJs, false) => { // mark as external Ok(ResolveResultOption::some( - ResolveResult::primary(ResolveResultItem::External( - request_str.into(), - ExternalType::CommonJs, - )) + ResolveResult::primary(ResolveResultItem::External { + name: request_str.into(), + ty: ExternalType::CommonJs, + // TODO(arlyon): wiring up in a follow up PR + traced: ExternalTraced::Untraced, + }) .cell(), )) } @@ -412,14 +414,16 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { } else { // mark as external Ok(ResolveResultOption::some( - ResolveResult::primary(ResolveResultItem::External( - request_str.into(), - if resolves_equal { + ResolveResult::primary(ResolveResultItem::External { + name: request_str.into(), + ty: if resolves_equal { ExternalType::CommonJs } else { ExternalType::EcmaScriptModule }, - )) + // TODO(arlyon): wiring up in a follow up PR + traced: ExternalTraced::Untraced, + }) .cell(), )) } @@ -427,10 +431,12 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { (FileType::EcmaScriptModule, true) => { // mark as external Ok(ResolveResultOption::some( - ResolveResult::primary(ResolveResultItem::External( - request_str.into(), - ExternalType::EcmaScriptModule, - )) + ResolveResult::primary(ResolveResultItem::External { + name: request_str.into(), + ty: ExternalType::EcmaScriptModule, + // TODO(arlyon): wiring up in a follow up PR + traced: ExternalTraced::Untraced, + }) .cell(), )) } diff --git a/crates/next-core/src/next_shared/resolve.rs b/crates/next-core/src/next_shared/resolve.rs index 21c650675587c1..b80171f4e3c7f9 100644 --- a/crates/next-core/src/next_shared/resolve.rs +++ b/crates/next-core/src/next_shared/resolve.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use anyhow::Result; use lazy_static::lazy_static; -use turbo_tasks::{RcStr, Value, Vc}; +use turbo_tasks::{RcStr, ResolvedVc, Value, Vc}; use turbo_tasks_fs::{glob::Glob, FileSystemPath}; use turbopack_core::{ diagnostics::DiagnosticExt, @@ -15,7 +15,7 @@ use turbopack_core::{ AfterResolvePlugin, AfterResolvePluginCondition, BeforeResolvePlugin, BeforeResolvePluginCondition, }, - ExternalType, ResolveResult, ResolveResultItem, ResolveResultOption, + ExternalTraced, ExternalType, ResolveResult, ResolveResultItem, ResolveResultOption, }, }; @@ -204,14 +204,14 @@ pub(crate) fn get_invalid_styled_jsx_resolve_plugin( #[turbo_tasks::value] pub(crate) struct NextExternalResolvePlugin { - root: Vc, + project_path: ResolvedVc, } #[turbo_tasks::value_impl] impl NextExternalResolvePlugin { #[turbo_tasks::function] - pub fn new(root: Vc) -> Vc { - NextExternalResolvePlugin { root }.cell() + pub fn new(project_path: ResolvedVc) -> Vc { + NextExternalResolvePlugin { project_path }.cell() } } @@ -220,7 +220,7 @@ impl AfterResolvePlugin for NextExternalResolvePlugin { #[turbo_tasks::function] fn after_resolve_condition(&self) -> Vc { AfterResolvePluginCondition::new( - self.root.root(), + self.project_path.root(), Glob::new("**/next/dist/**/*.{external,runtime.dev,runtime.prod}.js".into()), ) } @@ -233,18 +233,20 @@ impl AfterResolvePlugin for NextExternalResolvePlugin { _reference_type: Value, _request: Vc, ) -> Result> { - let raw_fs_path = &*fs_path.await?; - let path = raw_fs_path.path.to_string(); + let path = fs_path.await?.path.to_string(); // Find the starting index of 'next/dist' and slice from that point. It should // always be found since the glob pattern above is specific enough. let starting_index = path.find("next/dist").unwrap(); + let specifier = &path[starting_index..]; // Replace '/esm/' with '/' to match the CJS version of the file. - let modified_path = path[starting_index..].replace("/esm/", "/"); + let specifier: RcStr = specifier.replace("/esm/", "/").into(); + Ok(Vc::cell(Some( - ResolveResult::primary(ResolveResultItem::External( - modified_path.into(), - ExternalType::CommonJs, - )) + ResolveResult::primary(ResolveResultItem::External { + name: specifier.clone(), + ty: ExternalType::CommonJs, + traced: ExternalTraced::Traced(self.project_path), + }) .into(), ))) } diff --git a/crates/next-core/src/next_shared/webpack_rules/mod.rs b/crates/next-core/src/next_shared/webpack_rules/mod.rs index 3efc4f554a84ee..26a37f11e3f359 100644 --- a/crates/next-core/src/next_shared/webpack_rules/mod.rs +++ b/crates/next-core/src/next_shared/webpack_rules/mod.rs @@ -2,10 +2,10 @@ use anyhow::Result; use turbo_tasks::{RcStr, Vc}; use turbo_tasks_fs::FileSystemPath; use turbopack::module_options::WebpackLoadersOptions; -use turbopack_core::resolve::options::ImportMapping; +use turbopack_core::resolve::{options::ImportMapping, ExternalTraced, ExternalType}; use self::{babel::maybe_add_babel_loader, sass::maybe_add_sass_loader}; -use crate::{next_build::get_external_next_compiled_package_mapping, next_config::NextConfig}; +use crate::next_config::NextConfig; pub(crate) mod babel; pub(crate) mod sass; @@ -33,6 +33,13 @@ pub async fn webpack_loader_options( } #[turbo_tasks::function] -fn loader_runner_package_mapping() -> Vc { - get_external_next_compiled_package_mapping(Vc::cell("loader-runner".into())) +async fn loader_runner_package_mapping() -> Result> { + Ok(ImportMapping::Alternatives(vec![ImportMapping::External( + Some("next/dist/compiled/loader-runner".into()), + ExternalType::CommonJs, + ExternalTraced::Untraced, + None, + ) + .resolved_cell()]) + .cell()) } diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index afd4742289fb60..2d4feb4953aa10 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -33,11 +33,6 @@ export class ExternalObject { [K: symbol]: T } } -export interface TransformOutput { - code: string - map?: string - output?: string -} export function mdxCompile( value: string, option: Buffer, @@ -370,15 +365,6 @@ export interface NapiRewrite { has?: Array missing?: Array } -export function createTurboTasks( - outputPath: string, - persistentCaching: boolean, - memoryLimit?: number | undefined | null -): ExternalObject -export function runTurboTracing( - options: Buffer, - turboTasks: ExternalObject -): Promise> export function getTargetTriple(): string export function initHeapProfiler(): ExternalObject export function teardownHeapProfiler( diff --git a/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs b/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs index ae2d47ba4bc39f..c7273cb68cfd09 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs @@ -2,16 +2,20 @@ use std::collections::HashSet; use anyhow::Result; use auto_hash_map::AutoSet; +use futures::future::try_join_all; use turbo_tasks::{ FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc, }; +use turbo_tasks_fs::{FileSystem, VirtualFileSystem}; use super::{ availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItemInfo, chunk_content, chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkContentResult, ChunkItem, ChunkingContext, }; -use crate::{module::Module, output::OutputAssets, reference::ModuleReference}; +use crate::{ + module::Module, output::OutputAssets, rebase::RebasedAsset, reference::ModuleReference, +}; pub struct MakeChunkGroupResult { pub chunks: Vec>>, @@ -27,6 +31,7 @@ pub async fn make_chunk_group( let ChunkContentResult { chunk_items, async_modules, + traced_modules, external_module_references, forward_edges_inherit_async, local_back_edges_inherit_async, @@ -143,12 +148,24 @@ pub async fn make_chunk_group( .flat_map(|references| references.iter().copied()) .collect(); + let mut referenced_output_assets = references_to_output_assets(external_module_references) + .await? + .await? + .clone_value(); + + let rebased_modules = try_join_all(traced_modules.into_iter().map(|module| { + RebasedAsset::new(*module, module.ident().path().root(), traced_fs().root()).to_resolved() + })) + .await?; + + referenced_output_assets.extend(rebased_modules.into_iter().map(ResolvedVc::upcast)); + // Pass chunk items to chunking algorithm let mut chunks = make_chunks( chunking_context, Vc::cell(chunk_items.into_iter().collect()), "".into(), - references_to_output_assets(external_module_references).await?, + Vc::cell(referenced_output_assets), ) .await? .clone_value(); @@ -181,6 +198,12 @@ pub async fn make_chunk_group( }) } +// Without this wrapper, VirtualFileSystem::new_with_name always returns a new filesystem +#[turbo_tasks::function] +fn traced_fs() -> Vc { + VirtualFileSystem::new_with_name("traced".into()) +} + async fn references_to_output_assets( references: FxIndexSet>>, ) -> Result> { diff --git a/turbopack/crates/turbopack-core/src/chunk/mod.rs b/turbopack/crates/turbopack-core/src/chunk/mod.rs index 7d85afd17af72c..49cd4bea741257 100644 --- a/turbopack/crates/turbopack-core/src/chunk/mod.rs +++ b/turbopack/crates/turbopack-core/src/chunk/mod.rs @@ -16,7 +16,7 @@ use std::{ hash::Hash, }; -use anyhow::Result; +use anyhow::{bail, Result}; use auto_hash_map::AutoSet; use serde::{Deserialize, Serialize}; use tracing::{info_span, Span}; @@ -153,7 +153,17 @@ pub trait OutputChunk: Asset { /// Specifies how a chunk interacts with other chunks when building a chunk /// group #[derive( - Copy, Default, Clone, Hash, TraceRawVcs, Serialize, Deserialize, Eq, PartialEq, ValueDebugFormat, + Copy, + Debug, + Default, + Clone, + Hash, + TraceRawVcs, + Serialize, + Deserialize, + Eq, + PartialEq, + ValueDebugFormat, )] pub enum ChunkingType { /// Module is placed in the same chunk group and is loaded in parallel. It @@ -166,15 +176,18 @@ pub enum ChunkingType { /// An async loader is placed into the referencing chunk and loads the /// separate chunk group in which the module is placed. Async, - /// Module not placed in chunk group, but its references are still followed. + /// Module not placed in chunk group, but its references are still followed and placed into the + /// chunk group. Passthrough, + // Module not placed in chunk group, but its references are still followed. + Traced, } #[turbo_tasks::value(transparent)] pub struct ChunkingTypeOption(Option); -/// A [ModuleReference] implementing this trait and returning true for -/// [ChunkableModuleReference::is_chunkable] are considered as potentially +/// A [ModuleReference] implementing this trait and returning Some(_) for +/// [ChunkableModuleReference::chunking_type] are considered as potentially /// chunkable references. When all [Module]s of such a reference implement /// [ChunkableModule] they are placed in [Chunk]s during chunking. /// They are even potentially placed in the same [Chunk] when a chunk type @@ -191,6 +204,7 @@ type AsyncInfo = FxIndexMap>, Vec>>> pub struct ChunkContentResult { pub chunk_items: FxIndexSet>>, pub async_modules: FxIndexSet>>, + pub traced_modules: FxIndexSet>>, pub external_module_references: FxIndexSet>>, /// A map from local module to all children from which the async module /// status is inherited @@ -239,6 +253,10 @@ enum ChunkContentGraphNode { AsyncModule { module: Vc>, }, + // TODO docs + TracedModule { + module: Vc>, + }, // ModuleReferences that are not placed in the current chunk group ExternalModuleReference(ResolvedVc>), /// A list of directly referenced chunk items from which `is_async_module` @@ -369,6 +387,20 @@ async fn graph_node_to_referenced_nodes( .await? .into_iter() .map(|&module| async move { + if chunking_type == ChunkingType::Traced { + if *chunking_context.is_tracing_enabled().await? { + return Ok(( + Some(ChunkGraphEdge { + key: None, + node: ChunkContentGraphNode::TracedModule { module: *module }, + }), + None, + )); + } else { + return Ok((None, None)); + } + } + let Some(chunkable_module) = ResolvedVc::try_sidecast::>(module).await? else { @@ -465,6 +497,9 @@ async fn graph_node_to_referenced_nodes( )) } } + ChunkingType::Traced => { + bail!("unreachable ChunkingType::Traced"); + } } }) .try_join() @@ -621,10 +656,15 @@ async fn chunk_content_internal_parallel( let mut forward_edges_inherit_async = FxIndexMap::default(); let mut local_back_edges_inherit_async = FxIndexMap::default(); let mut available_async_modules_back_edges_inherit_async = FxIndexMap::default(); + let mut traced_modules = FxIndexSet::default(); for graph_node in graph_nodes { match graph_node { ChunkContentGraphNode::PassthroughChunkItem { .. } => {} + ChunkContentGraphNode::TracedModule { module } => { + let module = module.to_resolved().await?; + traced_modules.insert(module); + } ChunkContentGraphNode::ChunkItem { item, .. } => { chunk_items.insert(*item.to_resolved().await?); } @@ -662,6 +702,7 @@ async fn chunk_content_internal_parallel( Ok(ChunkContentResult { chunk_items, async_modules, + traced_modules, external_module_references, forward_edges_inherit_async, local_back_edges_inherit_async, diff --git a/turbopack/crates/turbopack-core/src/context.rs b/turbopack/crates/turbopack-core/src/context.rs index c587cd00beeed5..c36fc60af9d8e0 100644 --- a/turbopack/crates/turbopack-core/src/context.rs +++ b/turbopack/crates/turbopack-core/src/context.rs @@ -4,7 +4,8 @@ use turbo_tasks_fs::{glob::Glob, FileSystemPath}; use crate::{ compile_time_info::CompileTimeInfo, - module::Module, + issue::{module::ModuleIssue, IssueExt, StyledString}, + module::{Module, OptionModule}, reference_type::ReferenceType, resolve::{options::ResolveOptions, parse::Request, ModuleResolveResult, ResolveResult}, source::Source, @@ -15,6 +16,9 @@ pub enum ProcessResult { /// A module was created. Module(ResolvedVc>), + /// A module could not be created (according to the rules, e.g. no module type as assigned) + Unknown(Vc>), + /// Reference is ignored. This should lead to no module being included by /// the reference. Ignore, @@ -29,7 +33,39 @@ impl ProcessResult { ProcessResult::Ignore => { bail!("Expected process result to be a module, but it was ignored") } + ProcessResult::Unknown(_) => { + bail!("Expected process result to be a module, but it could not be processed") + } + } + } + + /// Unwrap the module, or return None and emit an issue + #[turbo_tasks::function] + pub async fn try_into_module(&self) -> Result> { + Ok(Vc::cell(match self { + ProcessResult::Module(module) => Some(*module), + ProcessResult::Unknown(source) => { + ProcessResult::emit_unknown_error(*source).await?; + None + } + ProcessResult::Ignore => None, + })) + } + + #[turbo_tasks::function] + pub fn emit_unknown_error(source: Vc>) { + ModuleIssue { + ident: source.ident(), + title: StyledString::Text("Unknown module type".into()).cell(), + description: StyledString::Text( + r"This module doesn't have an associated type. Use a known file extension, or register a loader for it. + + Read more: https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpack-loaders".into(), + ) + .cell(), } + .cell() + .emit(); } } @@ -72,6 +108,7 @@ pub trait AssetContext { self: Vc, result: Vc, reference_type: Value, + ignore_unknown: bool, ) -> Vc; /// Gets a new AssetContext with the transition applied. diff --git a/turbopack/crates/turbopack-core/src/introspect/utils.rs b/turbopack/crates/turbopack-core/src/introspect/utils.rs index f86773d8d0fdd4..00839fa5c73555 100644 --- a/turbopack/crates/turbopack-core/src/introspect/utils.rs +++ b/turbopack/crates/turbopack-core/src/introspect/utils.rs @@ -37,6 +37,11 @@ fn passthrough_reference_ty() -> Vc { Vc::cell("passthrough reference".into()) } +#[turbo_tasks::function] +fn traced_reference_ty() -> Vc { + Vc::cell("traced reference".into()) +} + #[turbo_tasks::function] pub async fn content_to_details(content: Vc) -> Result> { Ok(match &*content.await? { @@ -76,6 +81,7 @@ pub async fn children_from_module_references( } Some(ChunkingType::Async) => key = async_reference_ty(), Some(ChunkingType::Passthrough) => key = passthrough_reference_ty(), + Some(ChunkingType::Traced) => key = traced_reference_ty(), } } diff --git a/turbopack/crates/turbopack-core/src/issue/mod.rs b/turbopack/crates/turbopack-core/src/issue/mod.rs index b364fe27382f3d..abab6c516b3432 100644 --- a/turbopack/crates/turbopack-core/src/issue/mod.rs +++ b/turbopack/crates/turbopack-core/src/issue/mod.rs @@ -1,5 +1,6 @@ pub mod analyze; pub mod code_gen; +pub mod module; pub mod resolve; use std::{ diff --git a/turbopack/crates/turbopack-core/src/issue/module.rs b/turbopack/crates/turbopack-core/src/issue/module.rs new file mode 100644 index 00000000000000..059b28a5db94cf --- /dev/null +++ b/turbopack/crates/turbopack-core/src/issue/module.rs @@ -0,0 +1,35 @@ +use turbo_tasks::Vc; +use turbo_tasks_fs::FileSystemPath; + +use super::{Issue, IssueStage, OptionStyledString, StyledString}; +use crate::ident::AssetIdent; + +#[turbo_tasks::value(shared)] +pub struct ModuleIssue { + pub ident: Vc, + pub title: Vc, + pub description: Vc, +} + +#[turbo_tasks::value_impl] +impl Issue for ModuleIssue { + #[turbo_tasks::function] + fn stage(&self) -> Vc { + IssueStage::ProcessModule.cell() + } + + #[turbo_tasks::function] + fn file_path(&self) -> Vc { + self.ident.path() + } + + #[turbo_tasks::function] + fn title(&self) -> Vc { + self.title + } + + #[turbo_tasks::function] + fn description(&self) -> Vc { + Vc::cell(Some(self.description)) + } +} diff --git a/turbopack/crates/turbopack-core/src/issue/resolve.rs b/turbopack/crates/turbopack-core/src/issue/resolve.rs index e6f7061196935e..438949bfd959e5 100644 --- a/turbopack/crates/turbopack-core/src/issue/resolve.rs +++ b/turbopack/crates/turbopack-core/src/issue/resolve.rs @@ -34,17 +34,17 @@ impl Issue for ResolvingIssue { #[turbo_tasks::function] async fn title(&self) -> Result> { - let module_not_found = StyledString::Strong("Module not found".into()); - - Ok(match self.request.await?.request() { - Some(request) => StyledString::Line(vec![ - module_not_found, - StyledString::Text(": Can't resolve '".into()), - StyledString::Code(request), - StyledString::Text("'".into()), - ]), - None => module_not_found, - } + let request = self + .request + .request_pattern() + .to_string() + .await? + .clone_value(); + Ok(StyledString::Line(vec![ + StyledString::Strong("Module not found".into()), + StyledString::Text(": Can't resolve ".into()), + StyledString::Code(request), + ]) .cell()) } diff --git a/turbopack/crates/turbopack-core/src/lib.rs b/turbopack/crates/turbopack-core/src/lib.rs index ed5d26fd20df4b..169a27bc48a587 100644 --- a/turbopack/crates/turbopack-core/src/lib.rs +++ b/turbopack/crates/turbopack-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod package_json; pub mod proxied_asset; pub mod raw_module; pub mod raw_output; +pub mod rebase; pub mod reference; pub mod reference_type; pub mod resolve; diff --git a/turbopack/crates/turbopack/src/rebase/mod.rs b/turbopack/crates/turbopack-core/src/rebase.rs similarity index 98% rename from turbopack/crates/turbopack/src/rebase/mod.rs rename to turbopack/crates/turbopack-core/src/rebase.rs index c334652fc96930..1cc49dd6238bf9 100644 --- a/turbopack/crates/turbopack/src/rebase/mod.rs +++ b/turbopack/crates/turbopack-core/src/rebase.rs @@ -3,7 +3,8 @@ use std::hash::Hash; use anyhow::Result; use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks_fs::FileSystemPath; -use turbopack_core::{ + +use crate::{ asset::{Asset, AssetContent}, ident::AssetIdent, module::Module, diff --git a/turbopack/crates/turbopack-core/src/reference/mod.rs b/turbopack/crates/turbopack-core/src/reference/mod.rs index e40883cbe32e77..f843183b7b3e22 100644 --- a/turbopack/crates/turbopack-core/src/reference/mod.rs +++ b/turbopack/crates/turbopack-core/src/reference/mod.rs @@ -7,6 +7,7 @@ use turbo_tasks::{ }; use crate::{ + chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption}, issue::IssueDescriptionExt, module::{Module, Modules}, output::{OutputAsset, OutputAssets}, @@ -159,6 +160,45 @@ pub async fn referenced_modules_and_affecting_sources( Ok(Vc::cell(resolved_modules.into_iter().collect())) } +#[turbo_tasks::value] +pub struct TracedModuleReference { + module: ResolvedVc>, +} + +#[turbo_tasks::value_impl] +impl ModuleReference for TracedModuleReference { + #[turbo_tasks::function] + fn resolve_reference(&self) -> Vc { + ModuleResolveResult::module(self.module).cell() + } +} + +#[turbo_tasks::value_impl] +impl ValueToString for TracedModuleReference { + #[turbo_tasks::function] + async fn to_string(&self) -> Result> { + Ok(Vc::cell( + format!("traced {}", self.module.ident().to_string().await?).into(), + )) + } +} + +#[turbo_tasks::value_impl] +impl ChunkableModuleReference for TracedModuleReference { + #[turbo_tasks::function] + fn chunking_type(&self) -> Vc { + Vc::cell(Some(ChunkingType::Traced)) + } +} + +#[turbo_tasks::value_impl] +impl TracedModuleReference { + #[turbo_tasks::function] + pub fn new(module: ResolvedVc>) -> Vc { + Self::cell(TracedModuleReference { module }) + } +} + /// Aggregates all primary [Module]s referenced by an [Module]. [AssetReference] /// This does not include transitively references [Module]s, only includes /// primary [Module]s referenced. diff --git a/turbopack/crates/turbopack-core/src/resolve/mod.rs b/turbopack/crates/turbopack-core/src/resolve/mod.rs index dc80f934d7b784..7d1017270e212a 100644 --- a/turbopack/crates/turbopack-core/src/resolve/mod.rs +++ b/turbopack/crates/turbopack-core/src/resolve/mod.rs @@ -38,6 +38,7 @@ use crate::{ raw_module::RawModule, reference_type::ReferenceType, resolve::{ + node::{node_cjs_resolve_options, node_esm_resolve_options}, pattern::{read_matches, PatternMatch}, plugin::AfterResolvePlugin, }, @@ -65,7 +66,12 @@ use crate::{error::PrettyPrintError, issue::IssueSeverity}; pub enum ModuleResolveResultItem { Module(ResolvedVc>), OutputAsset(ResolvedVc>), - External(RcStr, ExternalType), + External { + /// uri, path, reference, etc. + name: RcStr, + ty: ExternalType, + traced: Option>, + }, Ignore, Error(ResolvedVc), Empty, @@ -350,6 +356,24 @@ impl ModuleResolveResult { } } +#[derive(Copy, Clone)] +#[turbo_tasks::value(shared)] +pub enum ExternalTraced { + Untraced, + Traced(ResolvedVc), +} + +impl ExternalTraced { + async fn as_string(&self) -> Result { + Ok(match self { + ExternalTraced::Untraced => "untraced".to_string(), + ExternalTraced::Traced(context) => { + format!("traced from {}", context.to_string().await?) + } + }) + } +} + #[derive( Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, TraceRawVcs, TaskInput, )] @@ -370,10 +394,15 @@ impl Display for ExternalType { } #[turbo_tasks::value(shared)] -#[derive(Clone, Debug)] +#[derive(Clone)] pub enum ResolveResultItem { Source(Vc>), - External(RcStr, ExternalType), + External { + /// uri, path, reference, etc. + name: RcStr, + ty: ExternalType, + traced: ExternalTraced, + }, Ignore, Error(Vc), Empty, @@ -424,7 +453,7 @@ impl RequestKey { } #[turbo_tasks::value(shared)] -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct ResolveResult { pub primary: FxIndexMap, pub affecting_sources: Vec>>, @@ -453,10 +482,14 @@ impl ValueToString for ResolveResult { ResolveResultItem::Source(a) => { result.push_str(&a.ident().to_string().await?); } - ResolveResultItem::External(s, ty) => { + ResolveResultItem::External { + name: s, + ty, + traced, + } => { result.push_str("external "); result.push_str(s); - write!(result, " ({})", ty)?; + write!(result, " ({}, {})", ty, traced.as_string().await?)?; } ResolveResultItem::Ignore => { result.push_str("ignore"); @@ -651,9 +684,17 @@ impl ResolveResult { request, match item { ResolveResultItem::Source(source) => asset_fn(source).await?, - ResolveResultItem::External(s, ty) => { - ModuleResolveResultItem::External(s, ty) - } + ResolveResultItem::External { + name, + ty, + // TODO remove this whole function? it's easy to drop traced + // externals now + traced: _, + } => ModuleResolveResultItem::External { + name, + ty, + traced: None, + }, ResolveResultItem::Ignore => ModuleResolveResultItem::Ignore, ResolveResultItem::Empty => ModuleResolveResultItem::Empty, ResolveResultItem::Error(e) => { @@ -674,6 +715,29 @@ impl ResolveResult { }) } + pub async fn map_items_module(&self, source_fn: A) -> Result + where + A: Fn(ResolveResultItem) -> AF, + AF: Future>, + { + Ok(ModuleResolveResult { + primary: self + .primary + .iter() + .map(|(request, item)| { + let asset_fn = &source_fn; + let request = request.clone(); + let item = item.clone(); + async move { Ok((request, asset_fn(item).await?)) } + }) + .try_join() + .await? + .into_iter() + .collect(), + affecting_sources: self.affecting_sources.clone(), + }) + } + /// Returns a new [ResolveResult] where all [RequestKey]s are set to the /// passed `request`. pub fn with_request_ref(&self, request: RcStr) -> Self { @@ -1455,9 +1519,10 @@ pub async fn url_resolve( } else { rel_result }; - let result = origin - .asset_context() - .process_resolve_result(result, reference_type.clone()); + let result = + origin + .asset_context() + .process_resolve_result(result, reference_type.clone(), false); handle_resolve_error( result, reference_type, @@ -1825,7 +1890,11 @@ async fn resolve_internal_inline( let uri: RcStr = format!("{}{}", protocol, remainder).into(); ResolveResult::primary_with_key( RequestKey::new(uri.clone()), - ResolveResultItem::External(uri, ExternalType::Url), + ResolveResultItem::External { + name: uri, + ty: ExternalType::Url, + traced: ExternalTraced::Untraced, + }, ) .into() } @@ -2524,6 +2593,48 @@ async fn resolve_import_map_result( )) } } + ImportMapResult::External(name, ty, traced, primary_alt) => { + let result = Some( + ResolveResult::primary(ResolveResultItem::External { + name: name.clone(), + ty: *ty, + traced: *traced, + }) + .cell(), + ); + + if let Some(context_dir) = primary_alt { + let request = Request::parse_string(name.clone()); + + // We must avoid cycles during resolving + if request.resolve().await? == *original_request + && context_dir.to_resolved().await? == original_lookup_path + { + None + } else { + let resolve_internal = resolve_internal( + **context_dir, + request, + match ty { + ExternalType::Url => options, + // TODO is that root correct? + ExternalType::CommonJs => node_cjs_resolve_options(context_dir.root()), + ExternalType::EcmaScriptModule => { + node_esm_resolve_options(context_dir.root()) + } + }, + ); + + if *resolve_internal.is_unresolvable().await? { + None + } else { + result + } + } + } else { + result + } + } ImportMapResult::Alternatives(list) => { let results = list .iter() diff --git a/turbopack/crates/turbopack-core/src/resolve/options.rs b/turbopack/crates/turbopack-core/src/resolve/options.rs index 695cc75812a11d..42db85b53ffb37 100644 --- a/turbopack/crates/turbopack-core/src/resolve/options.rs +++ b/turbopack/crates/turbopack-core/src/resolve/options.rs @@ -14,7 +14,7 @@ use super::{ plugin::BeforeResolvePlugin, AliasPattern, ExternalType, ResolveResult, ResolveResultItem, }; -use crate::resolve::{parse::Request, plugin::AfterResolvePlugin}; +use crate::resolve::{parse::Request, plugin::AfterResolvePlugin, ExternalTraced}; #[turbo_tasks::value(shared)] #[derive(Hash, Debug)] @@ -93,7 +93,15 @@ pub enum ResolveInPackage { #[turbo_tasks::value(shared)] #[derive(Clone)] pub enum ImportMapping { - External(Option, ExternalType), + // If specified, the optional name overrides the request, importing that external instead + // If the last option is a path, this behaves like PrimaryAlternative, only making it external + // if the request is resolvable from the directory. + External( + Option, + ExternalType, + ExternalTraced, + Option>, + ), /// An already resolved result that will be returned directly. Direct(ResolvedVc), /// A request alias that will be resolved first, and fall back to resolving @@ -111,7 +119,12 @@ pub enum ImportMapping { #[turbo_tasks::value(shared)] #[derive(Clone)] pub enum ReplacedImportMapping { - External(Option, ExternalType), + External( + Option, + ExternalType, + ExternalTraced, + Option>, + ), Direct(ResolvedVc), PrimaryAlternative(Pattern, Option>), Ignore, @@ -159,8 +172,8 @@ impl AliasTemplate for Vc { Box::pin(async move { let this = &*self.await?; Ok(match this { - ImportMapping::External(name, ty) => { - ReplacedImportMapping::External(name.clone(), *ty) + ImportMapping::External(name, ty, traced, primary_alt) => { + ReplacedImportMapping::External(name.clone(), *ty, *traced, *primary_alt) } ImportMapping::PrimaryAlternative(name, context) => { ReplacedImportMapping::PrimaryAlternative((*name).clone().into(), *context) @@ -188,14 +201,16 @@ impl AliasTemplate for Vc { Box::pin(async move { let this = &*self.await?; Ok(match this { - ImportMapping::External(name, ty) => { + ImportMapping::External(name, ty, traced, primary_alt) => { if let Some(name) = name { ReplacedImportMapping::External( capture.spread_into_star(name).as_string().map(|s| s.into()), *ty, + *traced, + *primary_alt, ) } else { - ReplacedImportMapping::External(None, *ty) + ReplacedImportMapping::External(None, *ty, *traced, *primary_alt) } } ImportMapping::PrimaryAlternative(name, context) => { @@ -323,9 +338,15 @@ pub struct ResolvedMap { } #[turbo_tasks::value(shared)] -#[derive(Clone, Debug)] +#[derive(Clone)] pub enum ImportMapResult { Result(ResolvedVc), + External( + RcStr, + ExternalType, + ExternalTraced, + Option>, + ), Alias(ResolvedVc, Option>), Alternatives(Vec), NoEntry, @@ -338,16 +359,20 @@ async fn import_mapping_to_result( ) -> Result { Ok(match &*mapping.await? { ReplacedImportMapping::Direct(result) => ImportMapResult::Result(*result), - ReplacedImportMapping::External(name, ty) => ImportMapResult::Result( - ResolveResult::primary(if let Some(name) = name { - ResolveResultItem::External(name.clone(), *ty) - } else if let Some(request) = request.await?.request() { - ResolveResultItem::External(request, *ty) - } else { - bail!("Cannot resolve external reference without request") - }) - .resolved_cell(), - ), + ReplacedImportMapping::External(name, ty, traced, primary_alt) => { + ImportMapResult::External( + if let Some(name) = name { + name.clone() + } else if let Some(request) = request.await?.request() { + request + } else { + bail!("Cannot resolve external reference without request") + }, + *ty, + *traced, + *primary_alt, + ) + } ReplacedImportMapping::Ignore => ImportMapResult::Result( ResolveResult::primary(ResolveResultItem::Ignore).resolved_cell(), ), @@ -379,6 +404,7 @@ impl ValueToString for ImportMapResult { async fn to_string(&self) -> Result> { match self { ImportMapResult::Result(_) => Ok(Vc::cell("Resolved by import map".into())), + ImportMapResult::External(_, _, _, _) => Ok(Vc::cell("TODO external".into())), ImportMapResult::Alias(request, context) => { let s = if let Some(path) = context { let path = path.to_string().await?; diff --git a/turbopack/crates/turbopack-core/src/resolve/pattern.rs b/turbopack/crates/turbopack-core/src/resolve/pattern.rs index 1f5b34013dac7f..665e2c77c06a63 100644 --- a/turbopack/crates/turbopack-core/src/resolve/pattern.rs +++ b/turbopack/crates/turbopack-core/src/resolve/pattern.rs @@ -9,7 +9,9 @@ use lazy_static::lazy_static; use regex::Regex; use serde::{Deserialize, Serialize}; use tracing::Instrument; -use turbo_tasks::{trace::TraceRawVcs, RcStr, ResolvedVc, Value, ValueToString, Vc}; +use turbo_tasks::{ + debug::ValueDebugFormat, trace::TraceRawVcs, RcStr, ResolvedVc, Value, ValueToString, Vc, +}; use turbo_tasks_fs::{ util::normalize_path, DirectoryContent, DirectoryEntry, FileSystemEntryType, FileSystemPath, LinkContent, LinkType, @@ -127,6 +129,16 @@ impl Pattern { } } + pub fn has_dynamic_parts(&self) -> bool { + match self { + Pattern::Constant(_) => false, + Pattern::Dynamic => true, + Pattern::Alternatives(list) | Pattern::Concatenation(list) => { + list.iter().any(|p| p.has_dynamic_parts()) + } + } + } + pub fn constant_prefix(&self) -> &str { // The normalized pattern is an Alternative of maximally merged // Concatenations, so extracting the first/only Concatenation child @@ -1257,7 +1269,7 @@ impl From for Pattern { impl Display for Pattern { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Pattern::Constant(c) => write!(f, "\"{c}\""), + Pattern::Constant(c) => write!(f, "'{c}'"), Pattern::Dynamic => write!(f, ""), Pattern::Alternatives(list) => write!( f, @@ -1287,7 +1299,7 @@ impl ValueToString for Pattern { } } -#[derive(Debug, PartialEq, Eq, Clone, TraceRawVcs, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Clone, TraceRawVcs, Serialize, Deserialize, ValueDebugFormat)] pub enum PatternMatch { File(RcStr, Vc), Directory(RcStr, Vc), diff --git a/turbopack/crates/turbopack-core/src/virtual_output.rs b/turbopack/crates/turbopack-core/src/virtual_output.rs index 21478f9410f3a2..6f1f31879652ff 100644 --- a/turbopack/crates/turbopack-core/src/virtual_output.rs +++ b/turbopack/crates/turbopack-core/src/virtual_output.rs @@ -1,24 +1,46 @@ +use anyhow::Result; use turbo_tasks::Vc; use turbo_tasks_fs::FileSystemPath; use crate::{ asset::{Asset, AssetContent}, ident::AssetIdent, - output::OutputAsset, + output::{OutputAsset, OutputAssets}, }; -/// An [OutputAsset] that is created from some passed source code. +/// An [OutputAsset] that is created from some passed source code and can have a list of references +/// to other assets. #[turbo_tasks::value] pub struct VirtualOutputAsset { pub path: Vc, pub content: Vc, + pub references: Vc, } #[turbo_tasks::value_impl] impl VirtualOutputAsset { #[turbo_tasks::function] pub fn new(path: Vc, content: Vc) -> Vc { - VirtualOutputAsset { path, content }.cell() + VirtualOutputAsset { + path, + content, + references: OutputAssets::empty(), + } + .cell() + } + + #[turbo_tasks::function] + pub fn new_with_references( + path: Vc, + content: Vc, + references: Vc, + ) -> Vc { + VirtualOutputAsset { + path, + content, + references, + } + .cell() } } @@ -28,6 +50,11 @@ impl OutputAsset for VirtualOutputAsset { fn ident(&self) -> Vc { AssetIdent::from_path(self.path) } + + #[turbo_tasks::function] + async fn references(&self) -> Result> { + Ok(self.references) + } } #[turbo_tasks::value_impl] diff --git a/turbopack/crates/turbopack-css/src/module_asset.rs b/turbopack/crates/turbopack-css/src/module_asset.rs index 757e43dcbe462c..59483a82345187 100644 --- a/turbopack/crates/turbopack-css/src/module_asset.rs +++ b/turbopack/crates/turbopack-css/src/module_asset.rs @@ -81,12 +81,12 @@ impl Module for ModuleCssAsset { .await? .iter() .copied() - .chain(match *self.inner().await? { - ProcessResult::Module(inner) => { - Some(Vc::upcast(InternalCssAssetReference::new(*inner))) - } - ProcessResult::Ignore => None, - }) + .chain( + self.inner() + .try_into_module() + .await? + .map(|inner| Vc::upcast(InternalCssAssetReference::new(*inner))), + ) .collect(); Ok(Vc::cell(references)) diff --git a/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs b/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs index 9636fa0890b6fd..8052b8a584a6f1 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs @@ -79,7 +79,9 @@ impl ReferencedAsset { } for (_key, result) in result.primary.iter() { match result { - ModuleResolveResultItem::External(request, ty) => { + ModuleResolveResultItem::External { + name: request, ty, .. + } => { return Ok(ReferencedAsset::External(request.clone(), *ty).cell()); } &ModuleResolveResultItem::Module(module) => { diff --git a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs index 214710cc5f4042..0713e0329ef1cc 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs @@ -2,14 +2,14 @@ use std::{fmt::Display, io::Write}; use anyhow::Result; use serde::{Deserialize, Serialize}; -use turbo_tasks::{trace::TraceRawVcs, RcStr, TaskInput, Vc}; +use turbo_tasks::{trace::TraceRawVcs, RcStr, ResolvedVc, TaskInput, Vc}; use turbo_tasks_fs::{glob::Glob, rope::RopeBuilder, FileContent, FileSystem, VirtualFileSystem}; use turbopack_core::{ asset::{Asset, AssetContent}, chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkableModule, ChunkingContext}, ident::AssetIdent, module::Module, - reference::ModuleReferences, + reference::{ModuleReference, ModuleReferences}, }; use crate::{ @@ -50,15 +50,21 @@ impl Display for CachedExternalType { pub struct CachedExternalModule { pub request: RcStr, pub external_type: CachedExternalType, + pub additional_references: Vec>>, } #[turbo_tasks::value_impl] impl CachedExternalModule { #[turbo_tasks::function] - pub fn new(request: RcStr, external_type: CachedExternalType) -> Vc { + pub fn new( + request: RcStr, + external_type: CachedExternalType, + additional_references: Vec>>, + ) -> Vc { Self::cell(CachedExternalModule { request, external_type, + additional_references, }) } @@ -103,7 +109,7 @@ impl Module for CachedExternalModule { fn ident(&self) -> Vc { let fs = VirtualFileSystem::new_with_name("externals".into()); - AssetIdent::from_path(fs.root()) + AssetIdent::from_path(fs.root().join(self.request.clone())) .with_layer(layer()) .with_modifier(Vc::cell(self.request.clone())) .with_modifier(Vc::cell(self.external_type.to_string().into())) @@ -179,6 +185,12 @@ pub struct CachedExternalModuleChunkItem { chunking_context: Vc>, } +// Without this wrapper, VirtualFileSystem::new_with_name always returns a new filesystem +#[turbo_tasks::function] +fn external_fs() -> Vc { + VirtualFileSystem::new_with_name("externals".into()) +} + #[turbo_tasks::value_impl] impl ChunkItem for CachedExternalModuleChunkItem { #[turbo_tasks::function] @@ -187,8 +199,15 @@ impl ChunkItem for CachedExternalModuleChunkItem { } #[turbo_tasks::function] - fn references(&self) -> Vc { - self.module.references() + async fn references(&self) -> Result> { + let additional_references = &self.module.await?.additional_references; + if !additional_references.is_empty() { + let mut module_references = self.module.references().await?.clone_value(); + module_references.extend(additional_references.iter().map(|rvc| **rvc)); + Ok(Vc::cell(module_references)) + } else { + Ok(self.module.references()) + } } #[turbo_tasks::function] diff --git a/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs b/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs index c01aedc6f0e623..947c7ef1f33bfc 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs @@ -313,7 +313,7 @@ async fn to_single_pattern_mapping( ) -> Result { let module = match resolve_item { ModuleResolveResultItem::Module(module) => *module, - ModuleResolveResultItem::External(s, ty) => { + ModuleResolveResultItem::External { name: s, ty, .. } => { return Ok(SinglePatternMapping::External(s.clone(), *ty)); } ModuleResolveResultItem::Ignore => return Ok(SinglePatternMapping::Ignored), diff --git a/turbopack/crates/turbopack-node/src/transforms/postcss.rs b/turbopack/crates/turbopack-node/src/transforms/postcss.rs index 5ba59b797ac670..ed4aeea7efa27c 100644 --- a/turbopack/crates/turbopack-node/src/transforms/postcss.rs +++ b/turbopack/crates/turbopack-node/src/transforms/postcss.rs @@ -203,18 +203,14 @@ async fn extra_configs_changed( .map(|path| async move { Ok( if matches!(&*path.get_type().await?, FileSystemEntryType::File) { - match *asset_context + asset_context .process( Vc::upcast(FileSource::new(path)), Value::new(ReferenceType::Internal(InnerAssets::empty())), ) + .try_into_module() .await? - { - ProcessResult::Module(module) => { - Some(any_content_changed_of_module(*module)) - } - ProcessResult::Ignore => None, - } + .map(|rvc| any_content_changed_of_module(*rvc)) } else { None }, diff --git a/turbopack/crates/turbopack-resolve/src/resolve.rs b/turbopack/crates/turbopack-resolve/src/resolve.rs index 152fad6d0ab951..898f3dab7e1ab6 100644 --- a/turbopack/crates/turbopack-resolve/src/resolve.rs +++ b/turbopack/crates/turbopack-resolve/src/resolve.rs @@ -7,7 +7,7 @@ use turbopack_core::resolve::{ ConditionValue, ImportMap, ImportMapping, ResolutionConditions, ResolveInPackage, ResolveIntoPackage, ResolveModules, ResolveOptions, }, - AliasMap, AliasPattern, ExternalType, FindContextFileResult, + AliasMap, AliasPattern, ExternalTraced, ExternalType, FindContextFileResult, }; use crate::{ @@ -106,11 +106,23 @@ async fn base_resolve_options( for req in NODE_EXTERNALS { direct_mappings.insert( AliasPattern::exact(req), - ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(), + ImportMapping::External( + None, + ExternalType::CommonJs, + ExternalTraced::Untraced, + None, + ) + .resolved_cell(), ); direct_mappings.insert( AliasPattern::exact(format!("node:{req}")), - ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(), + ImportMapping::External( + None, + ExternalType::CommonJs, + ExternalTraced::Untraced, + None, + ) + .resolved_cell(), ); } } @@ -118,12 +130,23 @@ async fn base_resolve_options( for req in EDGE_NODE_EXTERNALS { direct_mappings.insert( AliasPattern::exact(req), - ImportMapping::External(Some(format!("node:{req}").into()), ExternalType::CommonJs) - .resolved_cell(), + ImportMapping::External( + Some(format!("node:{req}").into()), + ExternalType::CommonJs, + ExternalTraced::Untraced, + None, + ) + .resolved_cell(), ); direct_mappings.insert( AliasPattern::exact(format!("node:{req}")), - ImportMapping::External(None, ExternalType::CommonJs).resolved_cell(), + ImportMapping::External( + None, + ExternalType::CommonJs, + ExternalTraced::Untraced, + None, + ) + .resolved_cell(), ); } } diff --git a/turbopack/crates/turbopack-resolve/src/typescript.rs b/turbopack/crates/turbopack-resolve/src/typescript.rs index b0a1f5cb372555..1e8a1fd74f11c0 100644 --- a/turbopack/crates/turbopack-resolve/src/typescript.rs +++ b/turbopack/crates/turbopack-resolve/src/typescript.rs @@ -447,11 +447,11 @@ pub async fn type_resolve( options, ) }; - let result = as_typings_result( - origin - .asset_context() - .process_resolve_result(result, ty.clone()), - ); + let result = as_typings_result(origin.asset_context().process_resolve_result( + result, + ty.clone(), + false, + )); handle_resolve_error( result, ty, diff --git a/turbopack/crates/turbopack-tests/tests/execution.rs b/turbopack/crates/turbopack-tests/tests/execution.rs index 2d3b674ab7c39f..1fc559d57d0a22 100644 --- a/turbopack/crates/turbopack-tests/tests/execution.rs +++ b/turbopack/crates/turbopack-tests/tests/execution.rs @@ -37,7 +37,7 @@ use turbopack_core::{ reference_type::{InnerAssets, ReferenceType}, resolve::{ options::{ImportMap, ImportMapping}, - ExternalType, + ExternalTraced, ExternalType, }, source::Source, }; @@ -274,7 +274,13 @@ async fn run_test(prepared_test: Vc) -> Result> let mut import_map = ImportMap::empty(); import_map.insert_wildcard_alias( "esm-external/", - ImportMapping::External(Some("*".into()), ExternalType::EcmaScriptModule).resolved_cell(), + ImportMapping::External( + Some("*".into()), + ExternalType::EcmaScriptModule, + ExternalTraced::Untraced, + None, + ) + .resolved_cell(), ); let asset_context: Vc> = Vc::upcast(ModuleAssetContext::new( diff --git a/turbopack/crates/turbopack/benches/node_file_trace.rs b/turbopack/crates/turbopack/benches/node_file_trace.rs index d2a23ebd06cffd..e6a07a231a36b2 100644 --- a/turbopack/crates/turbopack/benches/node_file_trace.rs +++ b/turbopack/crates/turbopack/benches/node_file_trace.rs @@ -8,7 +8,6 @@ use turbo_tasks_memory::MemoryBackend; use turbopack::{ emit_with_completion, module_options::{EcmascriptOptionsContext, ModuleOptionsContext}, - rebase::RebasedAsset, register, ModuleAssetContext, }; use turbopack_core::{ @@ -16,6 +15,7 @@ use turbopack_core::{ context::AssetContext, environment::{Environment, ExecutionEnvironment, NodeJsEnvironment}, file_source::FileSource, + rebase::RebasedAsset, reference_type::ReferenceType, }; use turbopack_resolve::resolve_options_context::ResolveOptionsContext; diff --git a/turbopack/crates/turbopack/examples/turbopack.rs b/turbopack/crates/turbopack/examples/turbopack.rs index 611fc9e5fddc22..e4bb6ef3cfc26f 100644 --- a/turbopack/crates/turbopack/examples/turbopack.rs +++ b/turbopack/crates/turbopack/examples/turbopack.rs @@ -13,12 +13,13 @@ use turbo_tasks::{ }; use turbo_tasks_fs::{DiskFileSystem, FileSystem}; use turbo_tasks_memory::MemoryBackend; -use turbopack::{emit_with_completion, rebase::RebasedAsset, register}; +use turbopack::{emit_with_completion, register}; use turbopack_core::{ compile_time_info::CompileTimeInfo, context::AssetContext, environment::{Environment, ExecutionEnvironment, NodeJsEnvironment}, file_source::FileSource, + rebase::RebasedAsset, PROJECT_FILESYSTEM_NAME, }; use turbopack_resolve::resolve_options_context::ResolveOptionsContext; diff --git a/turbopack/crates/turbopack/src/lib.rs b/turbopack/crates/turbopack/src/lib.rs index 40163e05664297..65ba329514f8be 100644 --- a/turbopack/crates/turbopack/src/lib.rs +++ b/turbopack/crates/turbopack/src/lib.rs @@ -11,7 +11,6 @@ pub mod evaluate_context; mod graph; pub mod module_options; pub mod nft_json; -pub mod rebase; pub mod transition; pub(crate) mod unsupported_sass; @@ -725,7 +724,11 @@ impl AssetContext for ModuleAssetContext { request, resolve_options, ); - let mut result = self.process_resolve_result(result.resolve().await?, reference_type); + let mut result = self.process_resolve_result( + result.resolve().await?, + reference_type, + request.request_pattern().await?.has_dynamic_parts(), + ); if *self.is_types_resolving_enabled().await? { let types_result = type_resolve( @@ -744,6 +747,7 @@ impl AssetContext for ModuleAssetContext { self: Vc, result: Vc, reference_type: Value, + ignore_unknown: bool, ) -> Result> { let this = self.await?; let transition = this.transition; @@ -762,6 +766,12 @@ impl AssetContext for ModuleAssetContext { ProcessResult::Module(m) => { ModuleResolveResultItem::Module(ResolvedVc::upcast(m)) } + ProcessResult::Unknown(source) => { + if !ignore_unknown { + ProcessResult::emit_unknown_error(source).await?; + } + ModuleResolveResultItem::Ignore + } ProcessResult::Ignore => ModuleResolveResultItem::Ignore, }) } @@ -961,7 +971,10 @@ pub async fn replace_externals( import_externals: bool, ) -> Result { for item in result.primary.values_mut() { - let ModuleResolveResultItem::External(request, ty) = item else { + let ModuleResolveResultItem::External { + name: request, ty, .. + } = item + else { continue; }; @@ -980,7 +993,7 @@ pub async fn replace_externals( } }; - let module = CachedExternalModule::new(request.clone(), external_type) + let module = CachedExternalModule::new(request.clone(), external_type, vec![]) .to_resolved() .await?; diff --git a/turbopack/crates/turbopack/src/transition/mod.rs b/turbopack/crates/turbopack/src/transition/mod.rs index 43e224c1fb4393..649220e036dc35 100644 --- a/turbopack/crates/turbopack/src/transition/mod.rs +++ b/turbopack/crates/turbopack/src/transition/mod.rs @@ -91,13 +91,16 @@ pub trait Transition { ) -> Result> { let asset = self.process_source(asset); let module_asset_context = self.process_context(module_asset_context); - let m = module_asset_context.process_default(asset, reference_type); - Ok(match *m.await? { + Ok(match &*module_asset_context + .process_default(asset, reference_type) + .await? + { ProcessResult::Module(m) => ProcessResult::Module( - self.process_module(*m, module_asset_context) + self.process_module(**m, module_asset_context) .to_resolved() .await?, ), + ProcessResult::Unknown(source) => ProcessResult::Unknown(*source), ProcessResult::Ignore => ProcessResult::Ignore, } .cell()) diff --git a/turbopack/crates/turbopack/tests/node-file-trace.rs b/turbopack/crates/turbopack/tests/node-file-trace.rs index fad297966c66dd..6d7db79c895bd4 100644 --- a/turbopack/crates/turbopack/tests/node-file-trace.rs +++ b/turbopack/crates/turbopack/tests/node-file-trace.rs @@ -37,7 +37,6 @@ use turbo_tasks_memory::MemoryBackend; use turbopack::{ emit_with_completion, module_options::{CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext}, - rebase::RebasedAsset, register, ModuleAssetContext, }; use turbopack_core::{ @@ -46,6 +45,7 @@ use turbopack_core::{ environment::{Environment, ExecutionEnvironment, NodeJsEnvironment}, file_source::FileSource, output::OutputAsset, + rebase::RebasedAsset, reference_type::ReferenceType, }; use turbopack_resolve::resolve_options_context::ResolveOptionsContext;