Skip to content

Commit

Permalink
Add moduleDetection compiler flag to allow for changing how modules a…
Browse files Browse the repository at this point in the history 
…re parsed (#47495)

* Add moduleDetection compiler flag to allow for changing how modules are parsed

The default setting is 'auto', where JSX containing files under react-jsx and react-jsxdev are
always parsed as modules, and esm-format files under module: node12+ are always parsed as modules,
in addition to the 'legacy' detection mode's conditions for other files. (Declaration files are exempt from
these new conditions)

The 'legacy' mode preserves TS's behavior prior to the introduction of this flag - a file is
parsed as a module if it contains an import, export, or import.meta expression.

In addition, there is a 'force' mode that forces all non-declaration files to be parsed as modules.
(Declaration files are still only modules if they contain a top-level import or export.)

This technically breaks the parser API, but it's kinda-sorta backwards compatible so long
as you don't need the functionality associated with more recent compiler flags.

* Fix post-merge lint

* Rename function

* Update default value documentation

* PR feedback

* Fix lint and typo
  • Loading branch information
@weswigham
weswigham committed Mar 11, 2022
1 parent 0271738 commit d1fa945
Show file tree
Hide file tree
Showing 144 changed files with 2,534 additions and 203 deletions.
12 changes: 12 additions & 0 deletions src/compiler/commandLineParser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1241,6 +1241,18 @@ namespace ts {
category: Diagnostics.Editor_Support,

},
{
name: "moduleDetection",
type: new Map(getEntries({
auto: ModuleDetectionKind.Auto,
legacy: ModuleDetectionKind.Legacy,
force: ModuleDetectionKind.Force,
})),
affectsModuleResolution: true,
description: Diagnostics.Control_what_method_is_used_to_detect_module_format_JS_files,
category: Diagnostics.Language_and_Environment,
defaultValueDescription: Diagnostics.auto_Colon_Treat_files_with_imports_exports_import_meta_jsx_with_jsx_Colon_react_jsx_or_esm_format_with_module_Colon_node12_as_modules,
}
];

/* @internal */
Expand Down
10 changes: 6 additions & 4 deletions src/compiler/core.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2156,14 +2156,16 @@ namespace ts {
return (arg: T) => f(arg) && g(arg);
}

export function or<T extends unknown[]>(...fs: ((...args: T) => boolean)[]): (...args: T) => boolean {
export function or<T extends unknown[], U>(...fs: ((...args: T) => U)[]): (...args: T) => U {
return (...args) => {
let lastResult: U;
for (const f of fs) {
if (f(...args)) {
return true;
lastResult = f(...args);
if (lastResult) {
return lastResult;
}
}
return false;
return lastResult!;
};
}

Expand Down
8 changes: 8 additions & 0 deletions src/compiler/diagnosticMessages.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1449,6 +1449,14 @@
"category": "Error",
"code": 1474
},
"Control what method is used to detect module-format JS files.": {
"category": "Message",
"code": 1475
},
"\"auto\": Treat files with imports, exports, import.meta, jsx (with jsx: react-jsx), or esm format (with module: node12+) as modules.": {
"category": "Message",
"code": 1476
},

"The types of '{0}' are incompatible between these types.": {
"category": "Error",
Expand Down
157 changes: 100 additions & 57 deletions src/compiler/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,41 @@ namespace ts {
text.charCodeAt(start + 3) !== CharacterCodes.slash;
}

/*@internal*/
export function isFileProbablyExternalModule(sourceFile: SourceFile) {
// Try to use the first top-level import/export when available, then
// fall back to looking for an 'import.meta' somewhere in the tree if necessary.
return forEach(sourceFile.statements, isAnExternalModuleIndicatorNode) ||
getImportMetaIfNecessary(sourceFile);
}

function isAnExternalModuleIndicatorNode(node: Node) {
return hasModifierOfKind(node, SyntaxKind.ExportKeyword)
|| isImportEqualsDeclaration(node) && isExternalModuleReference(node.moduleReference)
|| isImportDeclaration(node)
|| isExportAssignment(node)
|| isExportDeclaration(node) ? node : undefined;
}

function getImportMetaIfNecessary(sourceFile: SourceFile) {
return sourceFile.flags & NodeFlags.PossiblyContainsImportMeta ?
walkTreeForImportMeta(sourceFile) :
undefined;
}

function walkTreeForImportMeta(node: Node): Node | undefined {
return isImportMeta(node) ? node : forEachChild(node, walkTreeForImportMeta);
}

/** Do not use hasModifier inside the parser; it relies on parent pointers. Use this instead. */
function hasModifierOfKind(node: Node, kind: SyntaxKind) {
return some(node.modifiers, m => m.kind === kind);
}

function isImportMeta(node: Node): boolean {
return isMetaProperty(node) && node.keywordToken === SyntaxKind.ImportKeyword && node.name.escapedText === "meta";
}

/**
* Invokes a callback for each child of the given node. The 'cbNode' callback is invoked for all child nodes
* stored in properties. If a 'cbNodes' callback is specified, it is invoked for embedded arrays; otherwise,
Expand Down Expand Up @@ -642,17 +677,46 @@ namespace ts {
}
}

export function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes = false, scriptKind?: ScriptKind): SourceFile {
export interface CreateSourceFileOptions {
languageVersion: ScriptTarget;
/**
* Controls the format the file is detected as - this can be derived from only the path
* and files on disk, but needs to be done with a module resolution cache in scope to be performant.
* This is usually `undefined` for compilations that do not have `moduleResolution` values of `node12` or `nodenext`.
*/
impliedNodeFormat?: ModuleKind.ESNext | ModuleKind.CommonJS;
/**
* Controls how module-y-ness is set for the given file. Usually the result of calling
* `getSetExternalModuleIndicator` on a valid `CompilerOptions` object. If not present, the default
* check specified by `isFileProbablyExternalModule` will be used to set the field.
*/
setExternalModuleIndicator?: (file: SourceFile) => void;
}

function setExternalModuleIndicator(sourceFile: SourceFile) {
sourceFile.externalModuleIndicator = isFileProbablyExternalModule(sourceFile);
}

export function createSourceFile(fileName: string, sourceText: string, languageVersionOrOptions: ScriptTarget | CreateSourceFileOptions, setParentNodes = false, scriptKind?: ScriptKind): SourceFile {
tracing?.push(tracing.Phase.Parse, "createSourceFile", { path: fileName }, /*separateBeginAndEnd*/ true);
performance.mark("beforeParse");
let result: SourceFile;

perfLogger.logStartParseSourceFile(fileName);
const {
languageVersion,
setExternalModuleIndicator: overrideSetExternalModuleIndicator,
impliedNodeFormat: format
} = typeof languageVersionOrOptions === "object" ? languageVersionOrOptions : ({ languageVersion: languageVersionOrOptions } as CreateSourceFileOptions);
if (languageVersion === ScriptTarget.JSON) {
result = Parser.parseSourceFile(fileName, sourceText, languageVersion, /*syntaxCursor*/ undefined, setParentNodes, ScriptKind.JSON);
result = Parser.parseSourceFile(fileName, sourceText, languageVersion, /*syntaxCursor*/ undefined, setParentNodes, ScriptKind.JSON, noop);
}
else {
result = Parser.parseSourceFile(fileName, sourceText, languageVersion, /*syntaxCursor*/ undefined, setParentNodes, scriptKind);
const setIndicator = format === undefined ? overrideSetExternalModuleIndicator : (file: SourceFile) => {
file.impliedNodeFormat = format;
return (overrideSetExternalModuleIndicator || setExternalModuleIndicator)(file);
};
result = Parser.parseSourceFile(fileName, sourceText, languageVersion, /*syntaxCursor*/ undefined, setParentNodes, scriptKind, setIndicator);
}
perfLogger.logStopParseSourceFile();

Expand Down Expand Up @@ -851,7 +915,7 @@ namespace ts {
// attached to the EOF token.
let parseErrorBeforeNextFinishedNode = false;

export function parseSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, syntaxCursor: IncrementalParser.SyntaxCursor | undefined, setParentNodes = false, scriptKind?: ScriptKind): SourceFile {
export function parseSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, syntaxCursor: IncrementalParser.SyntaxCursor | undefined, setParentNodes = false, scriptKind?: ScriptKind, setExternalModuleIndicatorOverride?: (file: SourceFile) => void): SourceFile {
scriptKind = ensureScriptKind(fileName, scriptKind);
if (scriptKind === ScriptKind.JSON) {
const result = parseJsonText(fileName, sourceText, languageVersion, syntaxCursor, setParentNodes);
Expand All @@ -867,7 +931,7 @@ namespace ts {

initializeState(fileName, sourceText, languageVersion, syntaxCursor, scriptKind);

const result = parseSourceFileWorker(languageVersion, setParentNodes, scriptKind);
const result = parseSourceFileWorker(languageVersion, setParentNodes, scriptKind, setExternalModuleIndicatorOverride || setExternalModuleIndicator);

clearState();

Expand Down Expand Up @@ -955,7 +1019,7 @@ namespace ts {
}

// Set source file so that errors will be reported with this file name
const sourceFile = createSourceFile(fileName, ScriptTarget.ES2015, ScriptKind.JSON, /*isDeclaration*/ false, statements, endOfFileToken, sourceFlags);
const sourceFile = createSourceFile(fileName, ScriptTarget.ES2015, ScriptKind.JSON, /*isDeclaration*/ false, statements, endOfFileToken, sourceFlags, noop);

if (setParentNodes) {
fixupParentReferences(sourceFile);
Expand Down Expand Up @@ -1039,7 +1103,7 @@ namespace ts {
topLevel = true;
}

function parseSourceFileWorker(languageVersion: ScriptTarget, setParentNodes: boolean, scriptKind: ScriptKind): SourceFile {
function parseSourceFileWorker(languageVersion: ScriptTarget, setParentNodes: boolean, scriptKind: ScriptKind, setExternalModuleIndicator: (file: SourceFile) => void): SourceFile {
const isDeclarationFile = isDeclarationFileName(fileName);
if (isDeclarationFile) {
contextFlags |= NodeFlags.Ambient;
Expand All @@ -1054,7 +1118,7 @@ namespace ts {
Debug.assert(token() === SyntaxKind.EndOfFileToken);
const endOfFileToken = addJSDocComment(parseTokenNode<EndOfFileToken>());

const sourceFile = createSourceFile(fileName, languageVersion, scriptKind, isDeclarationFile, statements, endOfFileToken, sourceFlags);
const sourceFile = createSourceFile(fileName, languageVersion, scriptKind, isDeclarationFile, statements, endOfFileToken, sourceFlags, setExternalModuleIndicator);

// A member of ReadonlyArray<T> isn't assignable to a member of T[] (and prevents a direct cast) - but this is where we set up those members so they can be readonly in the future
processCommentPragmas(sourceFile as {} as PragmaContext, sourceText);
Expand Down Expand Up @@ -1213,28 +1277,42 @@ namespace ts {
setParentRecursive(rootNode, /*incremental*/ true);
}

function createSourceFile(fileName: string, languageVersion: ScriptTarget, scriptKind: ScriptKind, isDeclarationFile: boolean, statements: readonly Statement[], endOfFileToken: EndOfFileToken, flags: NodeFlags): SourceFile {
function createSourceFile(
fileName: string,
languageVersion: ScriptTarget,
scriptKind: ScriptKind,
isDeclarationFile: boolean,
statements: readonly Statement[],
endOfFileToken: EndOfFileToken,
flags: NodeFlags,
setExternalModuleIndicator: (sourceFile: SourceFile) => void): SourceFile {
// code from createNode is inlined here so createNode won't have to deal with special case of creating source files
// this is quite rare comparing to other nodes and createNode should be as fast as possible
let sourceFile = factory.createSourceFile(statements, endOfFileToken, flags);
setTextRangePosWidth(sourceFile, 0, sourceText.length);
setExternalModuleIndicator(sourceFile);
setFields(sourceFile);

// If we parsed this as an external module, it may contain top-level await
if (!isDeclarationFile && isExternalModule(sourceFile) && sourceFile.transformFlags & TransformFlags.ContainsPossibleTopLevelAwait) {
sourceFile = reparseTopLevelAwait(sourceFile);
setFields(sourceFile);
}

sourceFile.text = sourceText;
sourceFile.bindDiagnostics = [];
sourceFile.bindSuggestionDiagnostics = undefined;
sourceFile.languageVersion = languageVersion;
sourceFile.fileName = fileName;
sourceFile.languageVariant = getLanguageVariant(scriptKind);
sourceFile.isDeclarationFile = isDeclarationFile;
sourceFile.scriptKind = scriptKind;

return sourceFile;

function setFields(sourceFile: SourceFile) {
sourceFile.text = sourceText;
sourceFile.bindDiagnostics = [];
sourceFile.bindSuggestionDiagnostics = undefined;
sourceFile.languageVersion = languageVersion;
sourceFile.fileName = fileName;
sourceFile.languageVariant = getLanguageVariant(scriptKind);
sourceFile.isDeclarationFile = isDeclarationFile;
sourceFile.scriptKind = scriptKind;

setExternalModuleIndicator(sourceFile);
sourceFile.setExternalModuleIndicator = setExternalModuleIndicator;
}
}

function setContextFlag(val: boolean, flag: NodeFlags) {
Expand Down Expand Up @@ -7575,41 +7653,6 @@ namespace ts {
return withJSDoc(finishNode(node, pos), hasJSDoc);
}

function setExternalModuleIndicator(sourceFile: SourceFile) {
// Try to use the first top-level import/export when available, then
// fall back to looking for an 'import.meta' somewhere in the tree if necessary.
sourceFile.externalModuleIndicator =
forEach(sourceFile.statements, isAnExternalModuleIndicatorNode) ||
getImportMetaIfNecessary(sourceFile);
}

function isAnExternalModuleIndicatorNode(node: Node) {
return hasModifierOfKind(node, SyntaxKind.ExportKeyword)
|| isImportEqualsDeclaration(node) && ts.isExternalModuleReference(node.moduleReference)
|| isImportDeclaration(node)
|| isExportAssignment(node)
|| isExportDeclaration(node) ? node : undefined;
}

function getImportMetaIfNecessary(sourceFile: SourceFile) {
return sourceFile.flags & NodeFlags.PossiblyContainsImportMeta ?
walkTreeForExternalModuleIndicators(sourceFile) :
undefined;
}

function walkTreeForExternalModuleIndicators(node: Node): Node | undefined {
return isImportMeta(node) ? node : forEachChild(node, walkTreeForExternalModuleIndicators);
}

/** Do not use hasModifier inside the parser; it relies on parent pointers. Use this instead. */
function hasModifierOfKind(node: Node, kind: SyntaxKind) {
return some(node.modifiers, m => m.kind === kind);
}

function isImportMeta(node: Node): boolean {
return isMetaProperty(node) && node.keywordToken === SyntaxKind.ImportKeyword && node.name.escapedText === "meta";
}

const enum ParsingContext {
SourceElements, // Elements in source file
BlockStatements, // Statements in block
Expand Down Expand Up @@ -7652,7 +7695,7 @@ namespace ts {
currentToken = scanner.scan();
const jsDocTypeExpression = parseJSDocTypeExpression();

const sourceFile = createSourceFile("file.js", ScriptTarget.Latest, ScriptKind.JS, /*isDeclarationFile*/ false, [], factory.createToken(SyntaxKind.EndOfFileToken), NodeFlags.None);
const sourceFile = createSourceFile("file.js", ScriptTarget.Latest, ScriptKind.JS, /*isDeclarationFile*/ false, [], factory.createToken(SyntaxKind.EndOfFileToken), NodeFlags.None, noop);
const diagnostics = attachFileToDiagnostics(parseDiagnostics, sourceFile);
if (jsDocDiagnostics) {
sourceFile.jsDocDiagnostics = attachFileToDiagnostics(jsDocDiagnostics, sourceFile);
Expand Down Expand Up @@ -8698,7 +8741,7 @@ namespace ts {
if (sourceFile.statements.length === 0) {
// If we don't have any statements in the current source file, then there's no real
// way to incrementally parse. So just do a full parse instead.
return Parser.parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion, /*syntaxCursor*/ undefined, /*setParentNodes*/ true, sourceFile.scriptKind);
return Parser.parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion, /*syntaxCursor*/ undefined, /*setParentNodes*/ true, sourceFile.scriptKind, sourceFile.setExternalModuleIndicator);
}

// Make sure we're not trying to incrementally update a source file more than once. Once
Expand Down Expand Up @@ -8762,7 +8805,7 @@ namespace ts {
// inconsistent tree. Setting the parents on the new tree should be very fast. We
// will immediately bail out of walking any subtrees when we can see that their parents
// are already correct.
const result = Parser.parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion, syntaxCursor, /*setParentNodes*/ true, sourceFile.scriptKind);
const result = Parser.parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion, syntaxCursor, /*setParentNodes*/ true, sourceFile.scriptKind, sourceFile.setExternalModuleIndicator);
result.commentDirectives = getNewCommentDirectives(
sourceFile.commentDirectives,
result.commentDirectives,
Expand Down
Loading

0 comments on commit d1fa945

Please sign in to comment.