diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index b250c96813b4cd..828b469a962794 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -1,49 +1,38 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. // TODO(ry) Combine this implementation with //deno_typescript/compiler_main.js -// - +// these are imported for their side effects import "./globals.ts"; import "./ts_global.d.ts"; -import { buildBundle, setRootExports } from "./bundler.ts"; -import { bold, cyan, yellow } from "./colors.ts"; -import { CompilerOptions, TranspileOnlyResult } from "./compiler_api.ts"; -import { Console } from "./console.ts"; -import { core } from "./core.ts"; +import { TranspileOnlyResult } from "./compiler_api.ts"; +import { setRootExports } from "./compiler_bundler.ts"; +import { + defaultBundlerOptions, + defaultRuntimeCompileOptions, + defaultTranspileOptions, + Host +} from "./compiler_host.ts"; +import { + processImports, + processLocalImports, + resolveModules +} from "./compiler_imports.ts"; +import { + createWriteFile, + CompilerRequestType, + convertCompilerOptions, + ignoredDiagnostics, + WriteFileState, + processConfigureResponse +} from "./compiler_util.ts"; import { Diagnostic } from "./diagnostics.ts"; import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts"; -import { cwd } from "./dir.ts"; -import * as dispatch from "./dispatch.ts"; -import { sendAsync, sendSync } from "./dispatch_json.ts"; -import { TextEncoder } from "./text_encoding.ts"; import * as os from "./os.ts"; -import { getMappedModuleName, parseTypeDirectives } from "./type_directives.ts"; -import { assert, notImplemented } from "./util.ts"; +import { assert } from "./util.ts"; import * as util from "./util.ts"; import { window as self } from "./window.ts"; import { postMessage, workerClose, workerMain } from "./workers.ts"; -import { writeFileSync } from "./write_file.ts"; - -// Warning! The values in this enum are duplicated in cli/msg.rs -// Update carefully! -enum MediaType { - JavaScript = 0, - JSX = 1, - TypeScript = 2, - TSX = 3, - Json = 4, - Wasm = 5, - Unknown = 6 -} - -// Warning! The values in this enum are duplicated in cli/msg.rs -// Update carefully! -enum CompilerRequestType { - Compile = 0, - RuntimeCompile = 1, - RuntimeTranspile = 2 -} interface CompilerRequestCompile { type: CompilerRequestType.Compile; @@ -76,872 +65,19 @@ type CompilerRequest = | CompilerRequestRuntimeCompile | CompilerRequestRuntimeTranspile; -interface ConfigureResponse { - ignoredOptions?: string[]; - diagnostics?: ts.Diagnostic[]; -} - -interface EmitResult { +/** The format of the result sent back when doing a compilation. */ +interface CompileResult { emitSkipped: boolean; diagnostics?: Diagnostic; } -type WriteFileCallback = ( - fileName: string, - data: string, - sourceFiles?: readonly ts.SourceFile[] -) => void; - -// Startup boilerplate. This is necessary because the compiler has its own -// snapshot. (It would be great if we could remove these things or centralize -// them somewhere else.) -const console = new Console(core.print); -self.console = console; - -// bootstrap the worker environment, this gets called as the isolate is setup -self.workerMain = workerMain; - // bootstrap the runtime environment, this gets called as the isolate is setup self.denoMain = function denoMain(compilerType?: string): void { os.start(true, compilerType || "TS"); }; -const ASSETS = "$asset$"; -const OUT_DIR = "$deno$"; - -/** Options that either do nothing in Deno, or would cause undesired behavior - * if modified. */ -const ignoredCompilerOptions: readonly string[] = [ - "allowSyntheticDefaultImports", - "baseUrl", - "build", - "composite", - "declaration", - "declarationDir", - "declarationMap", - "diagnostics", - "downlevelIteration", - "emitBOM", - "emitDeclarationOnly", - "esModuleInterop", - "extendedDiagnostics", - "forceConsistentCasingInFileNames", - "help", - "importHelpers", - "incremental", - "inlineSourceMap", - "inlineSources", - "init", - "isolatedModules", - "lib", - "listEmittedFiles", - "listFiles", - "mapRoot", - "maxNodeModuleJsDepth", - "module", - "moduleResolution", - "newLine", - "noEmit", - "noEmitHelpers", - "noEmitOnError", - "noLib", - "noResolve", - "out", - "outDir", - "outFile", - "paths", - "preserveSymlinks", - "preserveWatchOutput", - "pretty", - "rootDir", - "rootDirs", - "showConfig", - "skipDefaultLibCheck", - "skipLibCheck", - "sourceMap", - "sourceRoot", - "stripInternal", - "target", - "traceResolution", - "tsBuildInfoFile", - "types", - "typeRoots", - "version", - "watch" -]; - -const defaultBundlerOptions: ts.CompilerOptions = { - inlineSourceMap: false, - module: ts.ModuleKind.AMD, - outDir: undefined, - outFile: `${OUT_DIR}/bundle.js`, - // disabled until we have effective way to modify source maps - sourceMap: false -}; - -/** Default options used by the compiler Host when compiling. */ -const defaultCompileOptions: ts.CompilerOptions = { - allowJs: true, - allowNonTsExtensions: true, - // TODO(#3324) Enable strict mode for user code. - // strict: true, - checkJs: false, - esModuleInterop: true, - module: ts.ModuleKind.ESNext, - outDir: OUT_DIR, - resolveJsonModule: true, - sourceMap: true, - stripComments: true, - target: ts.ScriptTarget.ESNext, - jsx: ts.JsxEmit.React -}; - -const defaultRuntimeCompileOptions: ts.CompilerOptions = { - outDir: undefined -}; - -/** Default options used when doing a transpile only. */ -const defaultTranspileOptions: ts.CompilerOptions = { - esModuleInterop: true, - module: ts.ModuleKind.ESNext, - sourceMap: true, - scriptComments: true, - target: ts.ScriptTarget.ESNext -}; - -const encoder = new TextEncoder(); - -const CHAR_DOT = 46; /* . */ -const CHAR_FORWARD_SLASH = 47; /* / */ - -// Resolves . and .. elements in a path with directory names -function normalizeString( - path: string, - allowAboveRoot: boolean, - separator: string, - isPathSeparator: (code: number) => boolean -): string { - let res = ""; - let lastSegmentLength = 0; - let lastSlash = -1; - let dots = 0; - let code: number; - for (let i = 0, len = path.length; i <= len; ++i) { - if (i < len) code = path.charCodeAt(i); - else if (isPathSeparator(code!)) break; - else code = CHAR_FORWARD_SLASH; - - if (isPathSeparator(code)) { - if (lastSlash === i - 1 || dots === 1) { - // NOOP - } else if (lastSlash !== i - 1 && dots === 2) { - if ( - res.length < 2 || - lastSegmentLength !== 2 || - res.charCodeAt(res.length - 1) !== CHAR_DOT || - res.charCodeAt(res.length - 2) !== CHAR_DOT - ) { - if (res.length > 2) { - const lastSlashIndex = res.lastIndexOf(separator); - if (lastSlashIndex === -1) { - res = ""; - lastSegmentLength = 0; - } else { - res = res.slice(0, lastSlashIndex); - lastSegmentLength = res.length - 1 - res.lastIndexOf(separator); - } - lastSlash = i; - dots = 0; - continue; - } else if (res.length === 2 || res.length === 1) { - res = ""; - lastSegmentLength = 0; - lastSlash = i; - dots = 0; - continue; - } - } - if (allowAboveRoot) { - if (res.length > 0) res += `${separator}..`; - else res = ".."; - lastSegmentLength = 2; - } - } else { - if (res.length > 0) res += separator + path.slice(lastSlash + 1, i); - else res = path.slice(lastSlash + 1, i); - lastSegmentLength = i - lastSlash - 1; - } - lastSlash = i; - dots = 0; - } else if (code === CHAR_DOT && dots !== -1) { - ++dots; - } else { - dots = -1; - } - } - return res; -} - -function resolvePath(...pathSegments: string[]): string { - let resolvedPath = ""; - let resolvedAbsolute = false; - - for (let i = pathSegments.length - 1; i >= -1 && !resolvedAbsolute; i--) { - let path: string; - - if (i >= 0) path = pathSegments[i]; - else path = cwd(); - - // Skip empty entries - if (path.length === 0) { - continue; - } - - resolvedPath = `${path}/${resolvedPath}`; - resolvedAbsolute = path.charCodeAt(0) === CHAR_FORWARD_SLASH; - } - - // At this point the path should be resolved to a full absolute path, but - // handle relative paths to be safe (might happen when process.cwd() fails) - - // Normalize the path - resolvedPath = normalizeString( - resolvedPath, - !resolvedAbsolute, - "/", - code => code === CHAR_FORWARD_SLASH - ); - - if (resolvedAbsolute) { - if (resolvedPath.length > 0) return `/${resolvedPath}`; - else return "/"; - } else if (resolvedPath.length > 0) return resolvedPath; - else return "."; -} - -function resolveSpecifier(specifier: string, referrer: string): string { - if (!specifier.startsWith(".")) { - return specifier; - } - const pathParts = referrer.split("/"); - pathParts.pop(); - let path = pathParts.join("/"); - path = path.endsWith("/") ? path : `${path}/`; - return resolvePath(path, specifier); -} - -function convertCompilerOptions(str: string): ts.CompilerOptions { - const options: CompilerOptions = JSON.parse(str); - const out: Record = {}; - const keys = Object.keys(options) as Array; - for (const key of keys) { - switch (key) { - case "jsx": - const value = options[key]; - if (value === "preserve") { - out[key] = ts.JsxEmit.Preserve; - } else if (value === "react") { - out[key] = ts.JsxEmit.React; - } else { - out[key] = ts.JsxEmit.ReactNative; - } - break; - case "module": - switch (options[key]) { - case "amd": - out[key] = ts.ModuleKind.AMD; - break; - case "commonjs": - out[key] = ts.ModuleKind.CommonJS; - break; - case "es2015": - case "es6": - out[key] = ts.ModuleKind.ES2015; - break; - case "esnext": - out[key] = ts.ModuleKind.ESNext; - break; - case "none": - out[key] = ts.ModuleKind.None; - break; - case "system": - out[key] = ts.ModuleKind.System; - break; - case "umd": - out[key] = ts.ModuleKind.UMD; - break; - default: - throw new TypeError("Unexpected module type"); - } - break; - case "target": - switch (options[key]) { - case "es3": - out[key] = ts.ScriptTarget.ES3; - break; - case "es5": - out[key] = ts.ScriptTarget.ES5; - break; - case "es6": - case "es2015": - out[key] = ts.ScriptTarget.ES2015; - break; - case "es2016": - out[key] = ts.ScriptTarget.ES2016; - break; - case "es2017": - out[key] = ts.ScriptTarget.ES2017; - break; - case "es2018": - out[key] = ts.ScriptTarget.ES2018; - break; - case "es2019": - out[key] = ts.ScriptTarget.ES2019; - break; - case "es2020": - out[key] = ts.ScriptTarget.ES2020; - break; - case "esnext": - out[key] = ts.ScriptTarget.ESNext; - break; - default: - throw new TypeError("Unexpected emit target."); - } - default: - out[key] = options[key]; - } - } - return out as ts.CompilerOptions; -} - -/** An array of TypeScript diagnostic types we ignore. */ -const ignoredDiagnostics = [ - // TS1103: 'for-await-of' statement is only allowed within an async function - // or async generator. - 1103, - // TS1308: 'await' expression is only allowed within an async function. - 1308, - // TS2691: An import path cannot end with a '.ts' extension. Consider - // importing 'bad-module' instead. - 2691, - // TS5009: Cannot find the common subdirectory path for the input files. - 5009, - // TS5055: Cannot write file - // 'http://localhost:4545/tests/subdir/mt_application_x_javascript.j4.js' - // because it would overwrite input file. - 5055, - // TypeScript is overly opinionated that only CommonJS modules kinds can - // support JSON imports. Allegedly this was fixed in - // Microsoft/TypeScript#26825 but that doesn't seem to be working here, - // so we will ignore complaints about this compiler setting. - 5070 -]; - -/** The shape of the SourceFile that comes from the privileged side */ -interface SourceFileJson { - url: string; - filename: string; - mediaType: MediaType; - sourceCode: string; -} - -/** A self registering abstraction of source files. */ -class SourceFile { - extension!: ts.Extension; - filename!: string; - - /** An array of tuples which represent the imports for the source file. The - * first element is the one that will be requested at compile time, the - * second is the one that should be actually resolved. This provides the - * feature of type directives for Deno. */ - importedFiles?: Array<[string, string]>; - - mediaType!: MediaType; - processed = false; - sourceCode!: string; - tsSourceFile?: ts.SourceFile; - url!: string; - - constructor(json: SourceFileJson) { - if (SourceFile._moduleCache.has(json.url)) { - throw new TypeError("SourceFile already exists"); - } - Object.assign(this, json); - this.extension = getExtension(this.url, this.mediaType); - SourceFile._moduleCache.set(this.url, this); - } - - /** Cache the source file to be able to be retrieved by `moduleSpecifier` and - * `containingFile`. */ - cache(moduleSpecifier: string, containingFile?: string): void { - containingFile = containingFile || ""; - let innerCache = SourceFile._specifierCache.get(containingFile); - if (!innerCache) { - innerCache = new Map(); - SourceFile._specifierCache.set(containingFile, innerCache); - } - innerCache.set(moduleSpecifier, this); - } - - /** Process the imports for the file and return them. */ - imports(): Array<[string, string]> { - if (this.processed) { - throw new Error("SourceFile has already been processed."); - } - assert(this.sourceCode != null); - // we shouldn't process imports for files which contain the nocheck pragma - // (like bundles) - if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) { - util.log(`Skipping imports for "${this.filename}"`); - return []; - } - const preProcessedFileInfo = ts.preProcessFile(this.sourceCode, true, true); - this.processed = true; - const files = (this.importedFiles = [] as Array<[string, string]>); - - function process(references: ts.FileReference[]): void { - for (const { fileName } of references) { - files.push([fileName, fileName]); - } - } - - const { - importedFiles, - referencedFiles, - libReferenceDirectives, - typeReferenceDirectives - } = preProcessedFileInfo; - const typeDirectives = parseTypeDirectives(this.sourceCode); - if (typeDirectives) { - for (const importedFile of importedFiles) { - files.push([ - importedFile.fileName, - getMappedModuleName(importedFile, typeDirectives) - ]); - } - } else { - process(importedFiles); - } - process(referencedFiles); - process(libReferenceDirectives); - process(typeReferenceDirectives); - return files; - } - - /** A cache of all the source files which have been loaded indexed by the - * url. */ - private static _moduleCache: Map = new Map(); - - /** A cache of source files based on module specifiers and containing files - * which is used by the TypeScript compiler to resolve the url */ - private static _specifierCache: Map< - string, - Map - > = new Map(); - - /** Retrieve a `SourceFile` based on a `moduleSpecifier` and `containingFile` - * or return `undefined` if not preset. */ - static getUrl( - moduleSpecifier: string, - containingFile: string - ): string | undefined { - const containingCache = this._specifierCache.get(containingFile); - if (containingCache) { - const sourceFile = containingCache.get(moduleSpecifier); - return sourceFile && sourceFile.url; - } - return undefined; - } - - /** Retrieve a `SourceFile` based on a `url` */ - static get(url: string): SourceFile | undefined { - return this._moduleCache.get(url); - } -} - -function resolveModules(specifiers: string[], referrer?: string): string[] { - util.log("compiler::resolveModules", { specifiers, referrer }); - return sendSync(dispatch.OP_RESOLVE_MODULES, { specifiers, referrer }); -} - -/** Ops to Rust to resolve special static assets. */ -function fetchAsset(name: string): string { - return sendSync(dispatch.OP_FETCH_ASSET, { name }); -} - -/** Ops to Rust to resolve and fetch modules meta data. */ -function fetchSourceFiles( - specifiers: string[], - referrer?: string -): Promise { - util.log("compiler::fetchSourceFiles", { specifiers, referrer }); - return sendAsync(dispatch.OP_FETCH_SOURCE_FILES, { - specifiers, - referrer - }); -} - -function getMediaType(filename: string): MediaType { - const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename); - if (!maybeExtension) { - util.log(`!!! Could not identify valid extension: "${filename}"`); - return MediaType.Unknown; - } - const [, extension] = maybeExtension; - switch (extension.toLowerCase()) { - case "js": - return MediaType.JavaScript; - case "jsx": - return MediaType.JSX; - case "json": - return MediaType.Json; - case "ts": - return MediaType.TypeScript; - case "tsx": - return MediaType.TSX; - case "wasm": - return MediaType.Wasm; - default: - util.log(`!!! Unknown extension: "${extension}"`); - return MediaType.Unknown; - } -} - -function processLocalImports( - sources: Record, - specifiers: Array<[string, string]>, - referrer?: string -): string[] { - if (!specifiers.length) { - return []; - } - const moduleNames = specifiers.map( - referrer - ? ([, specifier]): string => resolveSpecifier(specifier, referrer) - : ([, specifier]): string => specifier - ); - util.log("moduleNames:", moduleNames); - for (let i = 0; i < moduleNames.length; i++) { - const moduleName = moduleNames[i]; - const sourceFile = - SourceFile.get(moduleName) || - new SourceFile({ - url: moduleName, - filename: moduleName, - sourceCode: sources[moduleName], - mediaType: getMediaType(moduleName) - }); - sourceFile.cache(specifiers[i][0], referrer); - if (!sourceFile.processed) { - processLocalImports(sources, sourceFile.imports(), sourceFile.url); - } - } - return moduleNames; -} - -/** Recursively process the imports of modules, generating `SourceFile`s of any - * imported files. - * - * Specifiers are supplied in an array of tuples where the first is the - * specifier that will be requested in the code and the second is the specifier - * that should be actually resolved. */ -async function processImports( - specifiers: Array<[string, string]>, - referrer?: string -): Promise { - if (!specifiers.length) { - return []; - } - const sources = specifiers.map(([, moduleSpecifier]) => moduleSpecifier); - const resolveSources = resolveModules(sources, referrer); - const sourceFiles = await fetchSourceFiles(resolveSources, referrer); - assert(sourceFiles.length === specifiers.length); - for (let i = 0; i < sourceFiles.length; i++) { - const sourceFileJson = sourceFiles[i]; - const sourceFile = - SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson); - sourceFile.cache(specifiers[i][0], referrer); - if (!sourceFile.processed) { - await processImports(sourceFile.imports(), sourceFile.url); - } - } - return sourceFiles.map(sf => sf.url); -} - -/** Cache the contents of a file on the trusted side. */ -function cache( - moduleId: string, - emittedFileName: string, - contents: string, - checkJs = false -): void { - util.log("compiler::cache", { moduleId, emittedFileName, checkJs }); - const sf = SourceFile.get(moduleId); - - if (sf) { - // NOTE: If it's a `.json` file we don't want to write it to disk. - // JSON files are loaded and used by TS compiler to check types, but we don't want - // to emit them to disk because output file is the same as input file. - if (sf.extension === ts.Extension.Json) { - return; - } - - // NOTE: JavaScript files are only cached to disk if `checkJs` - // option in on - if (sf.extension === ts.Extension.Js && !checkJs) { - return; - } - } - - if (emittedFileName.endsWith(".map")) { - // Source Map - sendSync(dispatch.OP_CACHE, { - extension: ".map", - moduleId, - contents - }); - } else if ( - emittedFileName.endsWith(".js") || - emittedFileName.endsWith(".json") - ) { - // Compiled JavaScript - sendSync(dispatch.OP_CACHE, { - extension: ".js", - moduleId, - contents - }); - } else { - assert(false, `Trying to cache unhandled file type "${emittedFileName}"`); - } -} - -function processConfigureResponse( - configResult: ConfigureResponse, - configPath: string -): ts.Diagnostic[] | undefined { - const { ignoredOptions, diagnostics } = configResult; - if (ignoredOptions) { - console.warn( - yellow(`Unsupported compiler options in "${configPath}"\n`) + - cyan(` The following options were ignored:\n`) + - ` ${ignoredOptions.map((value): string => bold(value)).join(", ")}` - ); - } - return diagnostics; -} - -/** Returns the TypeScript Extension enum for a given media type. */ -function getExtension(fileName: string, mediaType: MediaType): ts.Extension { - switch (mediaType) { - case MediaType.JavaScript: - return ts.Extension.Js; - case MediaType.JSX: - return ts.Extension.Jsx; - case MediaType.TypeScript: - return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts; - case MediaType.TSX: - return ts.Extension.Tsx; - case MediaType.Json: - return ts.Extension.Json; - case MediaType.Wasm: - // Custom marker for Wasm type. - return ts.Extension.Js; - case MediaType.Unknown: - default: - throw TypeError("Cannot resolve extension."); - } -} - -interface CompilerHostOptions { - bundle?: boolean; - writeFile: WriteFileCallback; -} - -class Host implements ts.CompilerHost { - private readonly _options = defaultCompileOptions; - - private _writeFile: WriteFileCallback; - - private _getAsset(filename: string): SourceFile { - const sourceFile = SourceFile.get(filename); - if (sourceFile) { - return sourceFile; - } - const url = filename.split("/").pop()!; - const assetName = url.includes(".") ? url : `${url}.d.ts`; - const sourceCode = fetchAsset(assetName); - return new SourceFile({ - url, - filename, - mediaType: MediaType.TypeScript, - sourceCode - }); - } - - /* Deno specific APIs */ - - /** Provides the `ts.HostCompiler` interface for Deno. */ - constructor(options: CompilerHostOptions) { - const { bundle = false, writeFile } = options; - this._writeFile = writeFile; - if (bundle) { - // options we need to change when we are generating a bundle - Object.assign(this._options, defaultBundlerOptions); - } - } - - /** Take a configuration string, parse it, and use it to merge with the - * compiler's configuration options. The method returns an array of compiler - * options which were ignored, or `undefined`. */ - configure(path: string, configurationText: string): ConfigureResponse { - util.log("compiler::host.configure", path); - assert(configurationText); - const { config, error } = ts.parseConfigFileTextToJson( - path, - configurationText - ); - if (error) { - return { diagnostics: [error] }; - } - const { options, errors } = ts.convertCompilerOptionsFromJson( - config.compilerOptions, - cwd() - ); - const ignoredOptions: string[] = []; - for (const key of Object.keys(options)) { - if ( - ignoredCompilerOptions.includes(key) && - (!(key in this._options) || options[key] !== this._options[key]) - ) { - ignoredOptions.push(key); - delete options[key]; - } - } - Object.assign(this._options, options); - return { - ignoredOptions: ignoredOptions.length ? ignoredOptions : undefined, - diagnostics: errors.length ? errors : undefined - }; - } - - /** Merge options into the host's current set of compiler options and return - * the merged set. */ - mergeOptions(...options: ts.CompilerOptions[]): ts.CompilerOptions { - Object.assign(this._options, ...options); - return Object.assign({}, this._options); - } - - /* TypeScript CompilerHost APIs */ - - fileExists(_fileName: string): boolean { - return notImplemented(); - } - - getCanonicalFileName(fileName: string): string { - return fileName; - } - - getCompilationSettings(): ts.CompilerOptions { - util.log("compiler::host.getCompilationSettings()"); - return this._options; - } - - getCurrentDirectory(): string { - return ""; - } - - getDefaultLibFileName(_options: ts.CompilerOptions): string { - return ASSETS + "/lib.deno_runtime.d.ts"; - } - - getNewLine(): string { - return "\n"; - } - - getSourceFile( - fileName: string, - languageVersion: ts.ScriptTarget, - onError?: (message: string) => void, - shouldCreateNewSourceFile?: boolean - ): ts.SourceFile | undefined { - util.log("compiler::host.getSourceFile", fileName); - try { - assert(!shouldCreateNewSourceFile); - const sourceFile = fileName.startsWith(ASSETS) - ? this._getAsset(fileName) - : SourceFile.get(fileName); - assert(sourceFile != null); - if (!sourceFile.tsSourceFile) { - sourceFile.tsSourceFile = ts.createSourceFile( - fileName, - sourceFile.sourceCode, - languageVersion - ); - } - return sourceFile!.tsSourceFile; - } catch (e) { - if (onError) { - onError(String(e)); - } else { - throw e; - } - return undefined; - } - } - - readFile(_fileName: string): string | undefined { - return notImplemented(); - } - - resolveModuleNames( - moduleNames: string[], - containingFile: string - ): Array { - util.log("compiler::host.resolveModuleNames", { - moduleNames, - containingFile - }); - return moduleNames.map(specifier => { - const url = SourceFile.getUrl(specifier, containingFile); - const sourceFile = specifier.startsWith(ASSETS) - ? this._getAsset(specifier) - : url - ? SourceFile.get(url) - : undefined; - if (!sourceFile) { - return undefined; - } - return { - resolvedFileName: sourceFile.url, - isExternalLibraryImport: specifier.startsWith(ASSETS), - extension: sourceFile.extension - }; - }); - } - - useCaseSensitiveFileNames(): boolean { - return true; - } - - writeFile( - fileName: string, - data: string, - _writeByteOrderMark: boolean, - onError?: (message: string) => void, - sourceFiles?: readonly ts.SourceFile[] - ): void { - util.log("compiler::host.writeFile", fileName); - try { - this._writeFile(fileName, data, sourceFiles); - } catch (e) { - if (onError) { - onError(String(e)); - } else { - throw e; - } - } - } -} +// bootstrap the worker environment, this gets called as the isolate is setup +self.workerMain = workerMain; // provide the "main" function that will be called by the privileged side when // lazy instantiating the compiler web worker @@ -953,58 +89,40 @@ self.compilerMain = function compilerMain(): void { data: CompilerRequest; }): Promise => { switch (request.type) { + // `Compile` are requests from the internals to Deno, generated by both + // the `run` and `bundle` sub command. case CompilerRequestType.Compile: { - const { rootNames, configPath, config, outFile, bundle } = request; + const { bundle, config, configPath, outFile, rootNames } = request; util.log(">>> compile start", { rootNames, type: CompilerRequestType[request.type] }); - // This will recursively analyse all the code for other imports, requesting - // those from the privileged side, populating the in memory cache which - // will be used by the host, before resolving. + // This will recursively analyse all the code for other imports, + // requesting those from the privileged side, populating the in memory + // cache which will be used by the host, before resolving. const resolvedRootModules = await processImports( rootNames.map(rootName => [rootName, rootName]) ); - const writeFile = ( - fileName: string, - data: string, - sourceFiles?: readonly ts.SourceFile[] - ): void => { - assert(sourceFiles != null); - if (!bundle) { - assert(sourceFiles.length === 1); - cache( - sourceFiles[0].fileName, - fileName, - data, - host.getCompilationSettings().checkJs - ); - } else { - // if the fileName is set to an internal value, just noop - if (outFile && outFile.startsWith("$deno$")) { - return; - } - // we only support single root names for bundles - assert(rootNames.length === 1); - const out = buildBundle(rootNames[0], data, sourceFiles); - if (outFile) { - const encodedData = encoder.encode(out); - console.warn(`Emitting bundle to "${outFile}"`); - writeFileSync(outFile, encodedData); - console.warn( - `${util.humanFileSize(encodedData.length)} emitted.` - ); - } else { - console.log(out); - } - } + // When a programme is emitted, TypeScript will call `writeFile` with + // each file that needs to be emitted. The Deno compiler host delegates + // this, to make it easier to perform the right actions, which vary + // based a lot on the request. For a `Compile` request, we need to + // cache all the files in the privileged side if we aren't bundling, + // and if we are bundling we need to enrich the bundle and either write + // out the bundle or log it to the console. + const state: WriteFileState = { + type: request.type, + bundle, + host: undefined, + outFile, + rootNames }; + const writeFile = createWriteFile(state); - const host = new Host({ bundle, writeFile }); - - let diagnostics: ts.Diagnostic[] | undefined; + const host = (state.host = new Host({ bundle, writeFile })); + let diagnostics: readonly ts.Diagnostic[] | undefined; // if there is a configuration supplied, we need to parse that if (config && config.length && configPath) { @@ -1036,17 +154,16 @@ self.compilerMain = function compilerMain(): void { emitSkipped = emitResult.emitSkipped; // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned // without casting. - diagnostics = emitResult.diagnostics as ts.Diagnostic[]; + diagnostics = emitResult.diagnostics; } } - const result: EmitResult = { + const result: CompileResult = { emitSkipped, diagnostics: diagnostics.length ? fromTypeScriptDiagnostic(diagnostics) : undefined }; - postMessage(result); util.log("<<< compile end", { @@ -1056,6 +173,9 @@ self.compilerMain = function compilerMain(): void { break; } case CompilerRequestType.RuntimeCompile: { + // `RuntimeCompile` are requests from a runtime user, both compiles and + // bundles. The process is similar to a request from the privileged + // side, but also returns the output to the on message. const { rootName, sources, options, bundle } = request; util.log(">>> runtime compile start", { @@ -1072,36 +192,18 @@ self.compilerMain = function compilerMain(): void { ? processLocalImports(sources, [[resolvedRootName, resolvedRootName]]) : await processImports([[resolvedRootName, resolvedRootName]]); - const emitMap: Record = {}; - let emitBundle = ""; - const writeFile = ( - fileName: string, - data: string, - sourceFiles?: readonly ts.SourceFile[] - ): void => { - assert(sourceFiles != null); - if (!bundle) { - assert(sourceFiles.length === 1); - emitMap[fileName] = data; - // we only want to cache the compiler output if we are resolving - // modules externally - if (!sources) { - cache( - sourceFiles[0].fileName, - fileName, - data, - host.getCompilationSettings().checkJs - ); - } - } else { - // we only support single root names for bundles - assert(rootNames.length === 1); - emitBundle = buildBundle(rootNames[0], data, sourceFiles); - } + const state: WriteFileState = { + type: request.type, + bundle, + host: undefined, + rootNames, + sources, + emitMap: {}, + emitBundle: undefined }; + const writeFile = createWriteFile(state); - const host = new Host({ bundle, writeFile }); - + const host = (state.host = new Host({ bundle, writeFile })); const compilerOptions = [defaultRuntimeCompileOptions]; if (options) { compilerOptions.push(convertCompilerOptions(options)); @@ -1130,15 +232,16 @@ self.compilerMain = function compilerMain(): void { const { items } = fromTypeScriptDiagnostic(diagnostics); const result = [ items && items.length ? items : undefined, - bundle ? emitBundle : emitMap + bundle ? state.emitBundle : state.emitMap ]; postMessage(result); + assert(state.emitMap); util.log("<<< runtime compile finish", { rootName, sources: sources ? Object.keys(sources) : undefined, bundle, - emitMap: Object.keys(emitMap) + emitMap: Object.keys(state.emitMap) }); break; @@ -1153,6 +256,7 @@ self.compilerMain = function compilerMain(): void { convertCompilerOptions(options) ) : defaultTranspileOptions; + for (const [fileName, inputText] of Object.entries(sources)) { const { outputText: source, sourceMapText: map } = ts.transpileModule( inputText, @@ -1162,8 +266,8 @@ self.compilerMain = function compilerMain(): void { } ); result[fileName] = { source, map }; - postMessage(result); } + postMessage(result); break; } @@ -1202,10 +306,7 @@ self.wasmCompilerMain = function wasmCompilerMain(): void { new Set(WebAssembly.Module.exports(compiled).map(({ name }) => name)) ); - postMessage({ - importList, - exportList - }); + postMessage({ importList, exportList }); util.log("<<< WASM compile end"); diff --git a/cli/js/compiler_api_test.ts b/cli/js/compiler_api_test.ts index 64a1301830d1bd..2258febfada9b3 100644 --- a/cli/js/compiler_api_test.ts +++ b/cli/js/compiler_api_test.ts @@ -13,9 +13,10 @@ const { test(async function compilerApiCompileSources() { const [diagnostics, actual] = await compile("/foo.ts", { - "/foo.ts": `console.log("foo");\nexport {}\n` + "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`, + "/bar.ts": `export const bar = "bar";\n` }); assert(diagnostics == null); assert(actual); - assertEquals(Object.keys(actual).length, 2); + assertEquals(Object.keys(actual).length, 4); }); diff --git a/cli/js/bundler.ts b/cli/js/compiler_bundler.ts similarity index 88% rename from cli/js/bundler.ts rename to cli/js/compiler_bundler.ts index 2e452e1482d32e..69a6e31c9f8502 100644 --- a/cli/js/bundler.ts +++ b/cli/js/compiler_bundler.ts @@ -6,16 +6,21 @@ import { assert, commonPath } from "./util.ts"; const BUNDLE_LOADER = "bundle_loader.js"; +/** A loader of bundled modules that we will inline into any bundle outputs. */ let bundleLoader: string; +/** Local state of what the root exports are of a root module. */ let rootExports: string[] | undefined; -/** Given a fileName and the data, emit the file to the file system. */ +/** Given a root name, contents, and source files, enrich the data of the + * bundle with a loader and re-export the exports of the root name. */ export function buildBundle( rootName: string, data: string, sourceFiles: readonly ts.SourceFile[] ): string { + // we can only do this once we are bootstrapped and easiest way to do it is + // inline here if (!bundleLoader) { bundleLoader = sendSync(dispatch.OP_FETCH_ASSET, { name: BUNDLE_LOADER }); } diff --git a/cli/js/compiler_host.ts b/cli/js/compiler_host.ts new file mode 100644 index 00000000000000..89f5f506dd4e27 --- /dev/null +++ b/cli/js/compiler_host.ts @@ -0,0 +1,302 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +import { MediaType, SourceFile } from "./compiler_sourcefile.ts"; +import { OUT_DIR, WriteFileCallback } from "./compiler_util.ts"; +import { cwd } from "./dir.ts"; +import { sendSync } from "./dispatch_json.ts"; +import * as dispatch from "./dispatch.ts"; +import { assert, notImplemented } from "./util.ts"; +import * as util from "./util.ts"; + +export interface CompilerHostOptions { + bundle?: boolean; + writeFile: WriteFileCallback; +} + +export interface ConfigureResponse { + ignoredOptions?: string[]; + diagnostics?: ts.Diagnostic[]; +} + +const ASSETS = "$asset$"; + +/** Options that need to be used when generating a bundle (either trusted or + * runtime). */ +export const defaultBundlerOptions: ts.CompilerOptions = { + inlineSourceMap: false, + module: ts.ModuleKind.AMD, + outDir: undefined, + outFile: `${OUT_DIR}/bundle.js`, + // disabled until we have effective way to modify source maps + sourceMap: false +}; + +/** Default options used by the compiler Host when compiling. */ +export const defaultCompileOptions: ts.CompilerOptions = { + allowJs: true, + allowNonTsExtensions: true, + // TODO(#3324) Enable strict mode for user code. + // strict: true, + checkJs: false, + esModuleInterop: true, + module: ts.ModuleKind.ESNext, + outDir: OUT_DIR, + resolveJsonModule: true, + sourceMap: true, + stripComments: true, + target: ts.ScriptTarget.ESNext, + jsx: ts.JsxEmit.React +}; + +/** Options that need to be used when doing a runtime (non bundled) compilation */ +export const defaultRuntimeCompileOptions: ts.CompilerOptions = { + outDir: undefined +}; + +/** Default options used when doing a transpile only. */ +export const defaultTranspileOptions: ts.CompilerOptions = { + esModuleInterop: true, + module: ts.ModuleKind.ESNext, + sourceMap: true, + scriptComments: true, + target: ts.ScriptTarget.ESNext +}; + +/** Options that either do nothing in Deno, or would cause undesired behavior + * if modified. */ +const ignoredCompilerOptions: readonly string[] = [ + "allowSyntheticDefaultImports", + "baseUrl", + "build", + "composite", + "declaration", + "declarationDir", + "declarationMap", + "diagnostics", + "downlevelIteration", + "emitBOM", + "emitDeclarationOnly", + "esModuleInterop", + "extendedDiagnostics", + "forceConsistentCasingInFileNames", + "help", + "importHelpers", + "incremental", + "inlineSourceMap", + "inlineSources", + "init", + "isolatedModules", + "lib", + "listEmittedFiles", + "listFiles", + "mapRoot", + "maxNodeModuleJsDepth", + "module", + "moduleResolution", + "newLine", + "noEmit", + "noEmitHelpers", + "noEmitOnError", + "noLib", + "noResolve", + "out", + "outDir", + "outFile", + "paths", + "preserveSymlinks", + "preserveWatchOutput", + "pretty", + "rootDir", + "rootDirs", + "showConfig", + "skipDefaultLibCheck", + "skipLibCheck", + "sourceMap", + "sourceRoot", + "stripInternal", + "target", + "traceResolution", + "tsBuildInfoFile", + "types", + "typeRoots", + "version", + "watch" +]; + +export class Host implements ts.CompilerHost { + private readonly _options = defaultCompileOptions; + + private _writeFile: WriteFileCallback; + + private _getAsset(filename: string): SourceFile { + const sourceFile = SourceFile.get(filename); + if (sourceFile) { + return sourceFile; + } + const url = filename.split("/").pop()!; + const name = url.includes(".") ? url : `${url}.d.ts`; + const sourceCode = sendSync(dispatch.OP_FETCH_ASSET, { name }); + return new SourceFile({ + url, + filename, + mediaType: MediaType.TypeScript, + sourceCode + }); + } + + /* Deno specific APIs */ + + /** Provides the `ts.HostCompiler` interface for Deno. */ + constructor(options: CompilerHostOptions) { + const { bundle = false, writeFile } = options; + this._writeFile = writeFile; + if (bundle) { + // options we need to change when we are generating a bundle + Object.assign(this._options, defaultBundlerOptions); + } + } + + /** Take a configuration string, parse it, and use it to merge with the + * compiler's configuration options. The method returns an array of compiler + * options which were ignored, or `undefined`. */ + configure(path: string, configurationText: string): ConfigureResponse { + util.log("compiler::host.configure", path); + assert(configurationText); + const { config, error } = ts.parseConfigFileTextToJson( + path, + configurationText + ); + if (error) { + return { diagnostics: [error] }; + } + const { options, errors } = ts.convertCompilerOptionsFromJson( + config.compilerOptions, + cwd() + ); + const ignoredOptions: string[] = []; + for (const key of Object.keys(options)) { + if ( + ignoredCompilerOptions.includes(key) && + (!(key in this._options) || options[key] !== this._options[key]) + ) { + ignoredOptions.push(key); + delete options[key]; + } + } + Object.assign(this._options, options); + return { + ignoredOptions: ignoredOptions.length ? ignoredOptions : undefined, + diagnostics: errors.length ? errors : undefined + }; + } + + /** Merge options into the host's current set of compiler options and return + * the merged set. */ + mergeOptions(...options: ts.CompilerOptions[]): ts.CompilerOptions { + Object.assign(this._options, ...options); + return Object.assign({}, this._options); + } + + /* TypeScript CompilerHost APIs */ + + fileExists(_fileName: string): boolean { + return notImplemented(); + } + + getCanonicalFileName(fileName: string): string { + return fileName; + } + + getCompilationSettings(): ts.CompilerOptions { + util.log("compiler::host.getCompilationSettings()"); + return this._options; + } + + getCurrentDirectory(): string { + return ""; + } + + getDefaultLibFileName(_options: ts.CompilerOptions): string { + return ASSETS + "/lib.deno_runtime.d.ts"; + } + + getNewLine(): string { + return "\n"; + } + + getSourceFile( + fileName: string, + languageVersion: ts.ScriptTarget, + onError?: (message: string) => void, + shouldCreateNewSourceFile?: boolean + ): ts.SourceFile | undefined { + util.log("compiler::host.getSourceFile", fileName); + try { + assert(!shouldCreateNewSourceFile); + const sourceFile = fileName.startsWith(ASSETS) + ? this._getAsset(fileName) + : SourceFile.get(fileName); + assert(sourceFile != null); + if (!sourceFile.tsSourceFile) { + sourceFile.tsSourceFile = ts.createSourceFile( + fileName, + sourceFile.sourceCode, + languageVersion + ); + } + return sourceFile!.tsSourceFile; + } catch (e) { + if (onError) { + onError(String(e)); + } else { + throw e; + } + return undefined; + } + } + + readFile(_fileName: string): string | undefined { + return notImplemented(); + } + + resolveModuleNames( + moduleNames: string[], + containingFile: string + ): Array { + util.log("compiler::host.resolveModuleNames", { + moduleNames, + containingFile + }); + return moduleNames.map(specifier => { + const url = SourceFile.getUrl(specifier, containingFile); + const sourceFile = specifier.startsWith(ASSETS) + ? this._getAsset(specifier) + : url + ? SourceFile.get(url) + : undefined; + if (!sourceFile) { + return undefined; + } + return { + resolvedFileName: sourceFile.url, + isExternalLibraryImport: specifier.startsWith(ASSETS), + extension: sourceFile.extension + }; + }); + } + + useCaseSensitiveFileNames(): boolean { + return true; + } + + writeFile( + fileName: string, + data: string, + _writeByteOrderMark: boolean, + _onError?: (message: string) => void, + sourceFiles?: readonly ts.SourceFile[] + ): void { + util.log("compiler::host.writeFile", fileName); + this._writeFile(fileName, data, sourceFiles); + } +} diff --git a/cli/js/compiler_imports.ts b/cli/js/compiler_imports.ts new file mode 100644 index 00000000000000..9243b5752311e7 --- /dev/null +++ b/cli/js/compiler_imports.ts @@ -0,0 +1,251 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +import { + MediaType, + SourceFile, + SourceFileJson +} from "./compiler_sourcefile.ts"; +import { cwd } from "./dir.ts"; +import * as dispatch from "./dispatch.ts"; +import { sendAsync, sendSync } from "./dispatch_json.ts"; +import { assert } from "./util.ts"; +import * as util from "./util.ts"; + +// Constants used by `normalizeString` and `resolvePath` +const CHAR_DOT = 46; /* . */ +const CHAR_FORWARD_SLASH = 47; /* / */ + +/** Resolves `.` and `..` elements in a path with directory names */ +function normalizeString( + path: string, + allowAboveRoot: boolean, + separator: string, + isPathSeparator: (code: number) => boolean +): string { + let res = ""; + let lastSegmentLength = 0; + let lastSlash = -1; + let dots = 0; + let code: number; + for (let i = 0, len = path.length; i <= len; ++i) { + if (i < len) code = path.charCodeAt(i); + else if (isPathSeparator(code!)) break; + else code = CHAR_FORWARD_SLASH; + + if (isPathSeparator(code)) { + if (lastSlash === i - 1 || dots === 1) { + // NOOP + } else if (lastSlash !== i - 1 && dots === 2) { + if ( + res.length < 2 || + lastSegmentLength !== 2 || + res.charCodeAt(res.length - 1) !== CHAR_DOT || + res.charCodeAt(res.length - 2) !== CHAR_DOT + ) { + if (res.length > 2) { + const lastSlashIndex = res.lastIndexOf(separator); + if (lastSlashIndex === -1) { + res = ""; + lastSegmentLength = 0; + } else { + res = res.slice(0, lastSlashIndex); + lastSegmentLength = res.length - 1 - res.lastIndexOf(separator); + } + lastSlash = i; + dots = 0; + continue; + } else if (res.length === 2 || res.length === 1) { + res = ""; + lastSegmentLength = 0; + lastSlash = i; + dots = 0; + continue; + } + } + if (allowAboveRoot) { + if (res.length > 0) res += `${separator}..`; + else res = ".."; + lastSegmentLength = 2; + } + } else { + if (res.length > 0) res += separator + path.slice(lastSlash + 1, i); + else res = path.slice(lastSlash + 1, i); + lastSegmentLength = i - lastSlash - 1; + } + lastSlash = i; + dots = 0; + } else if (code === CHAR_DOT && dots !== -1) { + ++dots; + } else { + dots = -1; + } + } + return res; +} + +/** Resolve a path to the final path segment passed. */ +function resolvePath(...pathSegments: string[]): string { + let resolvedPath = ""; + let resolvedAbsolute = false; + + for (let i = pathSegments.length - 1; i >= -1 && !resolvedAbsolute; i--) { + let path: string; + + if (i >= 0) path = pathSegments[i]; + else path = cwd(); + + // Skip empty entries + if (path.length === 0) { + continue; + } + + resolvedPath = `${path}/${resolvedPath}`; + resolvedAbsolute = path.charCodeAt(0) === CHAR_FORWARD_SLASH; + } + + // At this point the path should be resolved to a full absolute path, but + // handle relative paths to be safe (might happen when cwd() fails) + + // Normalize the path + resolvedPath = normalizeString( + resolvedPath, + !resolvedAbsolute, + "/", + code => code === CHAR_FORWARD_SLASH + ); + + if (resolvedAbsolute) { + if (resolvedPath.length > 0) return `/${resolvedPath}`; + else return "/"; + } else if (resolvedPath.length > 0) return resolvedPath; + else return "."; +} + +/** Resolve a relative specifier based on the referrer. Used when resolving + * modules internally within the runtime compiler API. */ +function resolveSpecifier(specifier: string, referrer: string): string { + if (!specifier.startsWith(".")) { + return specifier; + } + const pathParts = referrer.split("/"); + pathParts.pop(); + let path = pathParts.join("/"); + path = path.endsWith("/") ? path : `${path}/`; + return resolvePath(path, specifier); +} + +/** Ops to Rust to resolve modules' URLs. */ +export function resolveModules( + specifiers: string[], + referrer?: string +): string[] { + util.log("compiler_imports::resolveModules", { specifiers, referrer }); + return sendSync(dispatch.OP_RESOLVE_MODULES, { specifiers, referrer }); +} + +/** Ops to Rust to fetch modules meta data. */ +function fetchSourceFiles( + specifiers: string[], + referrer?: string +): Promise { + util.log("compiler_imports::fetchSourceFiles", { specifiers, referrer }); + return sendAsync(dispatch.OP_FETCH_SOURCE_FILES, { + specifiers, + referrer + }); +} + +/** Given a filename, determine the media type based on extension. Used when + * resolving modules internally in a runtime compile. */ +function getMediaType(filename: string): MediaType { + const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename); + if (!maybeExtension) { + util.log(`!!! Could not identify valid extension: "${filename}"`); + return MediaType.Unknown; + } + const [, extension] = maybeExtension; + switch (extension.toLowerCase()) { + case "js": + return MediaType.JavaScript; + case "jsx": + return MediaType.JSX; + case "json": + return MediaType.Json; + case "ts": + return MediaType.TypeScript; + case "tsx": + return MediaType.TSX; + case "wasm": + return MediaType.Wasm; + default: + util.log(`!!! Unknown extension: "${extension}"`); + return MediaType.Unknown; + } +} + +/** Recursively process the imports of modules from within the supplied sources, + * generating `SourceFile`s of any imported files. + * + * Specifiers are supplied in an array of tuples where the first is the + * specifier that will be requested in the code and the second is the specifier + * that should be actually resolved. */ +export function processLocalImports( + sources: Record, + specifiers: Array<[string, string]>, + referrer?: string +): string[] { + if (!specifiers.length) { + return []; + } + const moduleNames = specifiers.map( + referrer + ? ([, specifier]): string => resolveSpecifier(specifier, referrer) + : ([, specifier]): string => specifier + ); + for (let i = 0; i < moduleNames.length; i++) { + const moduleName = moduleNames[i]; + console.log(moduleName, getMediaType(moduleName)); + const sourceFile = + SourceFile.get(moduleName) || + new SourceFile({ + url: moduleName, + filename: moduleName, + sourceCode: sources[moduleName], + mediaType: getMediaType(moduleName) + }); + sourceFile.cache(specifiers[i][0], referrer); + if (!sourceFile.processed) { + processLocalImports(sources, sourceFile.imports(), sourceFile.url); + } + } + return moduleNames; +} + +/** Recursively process the imports of modules, generating `SourceFile`s of any + * imported files. + * + * Specifiers are supplied in an array of tuples where the first is the + * specifier that will be requested in the code and the second is the specifier + * that should be actually resolved. */ +export async function processImports( + specifiers: Array<[string, string]>, + referrer?: string +): Promise { + if (!specifiers.length) { + return []; + } + const sources = specifiers.map(([, moduleSpecifier]) => moduleSpecifier); + const resolvedSources = resolveModules(sources, referrer); + const sourceFiles = await fetchSourceFiles(resolvedSources, referrer); + assert(sourceFiles.length === specifiers.length); + for (let i = 0; i < sourceFiles.length; i++) { + const sourceFileJson = sourceFiles[i]; + const sourceFile = + SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson); + sourceFile.cache(specifiers[i][0], referrer); + if (!sourceFile.processed) { + await processImports(sourceFile.imports(), sourceFile.url); + } + } + return resolvedSources; +} diff --git a/cli/js/compiler_sourcefile.ts b/cli/js/compiler_sourcefile.ts new file mode 100644 index 00000000000000..46e5cbe3b252b9 --- /dev/null +++ b/cli/js/compiler_sourcefile.ts @@ -0,0 +1,168 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +import { + getMappedModuleName, + parseTypeDirectives +} from "./compiler_type_directives.ts"; +import { assert, log } from "./util.ts"; + +// Warning! The values in this enum are duplicated in `cli/msg.rs` +// Update carefully! +export enum MediaType { + JavaScript = 0, + JSX = 1, + TypeScript = 2, + TSX = 3, + Json = 4, + Wasm = 5, + Unknown = 6 +} + +/** The shape of the SourceFile that comes from the privileged side */ +export interface SourceFileJson { + url: string; + filename: string; + mediaType: MediaType; + sourceCode: string; +} + +/** Returns the TypeScript Extension enum for a given media type. */ +function getExtension(fileName: string, mediaType: MediaType): ts.Extension { + switch (mediaType) { + case MediaType.JavaScript: + return ts.Extension.Js; + case MediaType.JSX: + return ts.Extension.Jsx; + case MediaType.TypeScript: + return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts; + case MediaType.TSX: + return ts.Extension.Tsx; + case MediaType.Json: + return ts.Extension.Json; + case MediaType.Wasm: + // Custom marker for Wasm type. + return ts.Extension.Js; + case MediaType.Unknown: + default: + throw TypeError("Cannot resolve extension."); + } +} + +/** A self registering abstraction of source files. */ +export class SourceFile { + extension!: ts.Extension; + filename!: string; + + /** An array of tuples which represent the imports for the source file. The + * first element is the one that will be requested at compile time, the + * second is the one that should be actually resolved. This provides the + * feature of type directives for Deno. */ + importedFiles?: Array<[string, string]>; + + mediaType!: MediaType; + processed = false; + sourceCode!: string; + tsSourceFile?: ts.SourceFile; + url!: string; + + constructor(json: SourceFileJson) { + if (SourceFile._moduleCache.has(json.url)) { + throw new TypeError("SourceFile already exists"); + } + Object.assign(this, json); + this.extension = getExtension(this.url, this.mediaType); + SourceFile._moduleCache.set(this.url, this); + } + + /** Cache the source file to be able to be retrieved by `moduleSpecifier` and + * `containingFile`. */ + cache(moduleSpecifier: string, containingFile?: string): void { + containingFile = containingFile || ""; + let innerCache = SourceFile._specifierCache.get(containingFile); + if (!innerCache) { + innerCache = new Map(); + SourceFile._specifierCache.set(containingFile, innerCache); + } + innerCache.set(moduleSpecifier, this); + } + + /** Process the imports for the file and return them. */ + imports(): Array<[string, string]> { + if (this.processed) { + throw new Error("SourceFile has already been processed."); + } + assert(this.sourceCode != null); + // we shouldn't process imports for files which contain the nocheck pragma + // (like bundles) + if (this.sourceCode.match(/\/{2}\s+@ts-nocheck/)) { + log(`Skipping imports for "${this.filename}"`); + return []; + } + const preProcessedFileInfo = ts.preProcessFile(this.sourceCode, true, true); + this.processed = true; + const files = (this.importedFiles = [] as Array<[string, string]>); + + function process(references: ts.FileReference[]): void { + for (const { fileName } of references) { + files.push([fileName, fileName]); + } + } + + const { + importedFiles, + referencedFiles, + libReferenceDirectives, + typeReferenceDirectives + } = preProcessedFileInfo; + const typeDirectives = parseTypeDirectives(this.sourceCode); + if (typeDirectives) { + for (const importedFile of importedFiles) { + files.push([ + importedFile.fileName, + getMappedModuleName(importedFile, typeDirectives) + ]); + } + } else { + process(importedFiles); + } + process(referencedFiles); + process(libReferenceDirectives); + process(typeReferenceDirectives); + return files; + } + + /** A cache of all the source files which have been loaded indexed by the + * url. */ + private static _moduleCache: Map = new Map(); + + /** A cache of source files based on module specifiers and containing files + * which is used by the TypeScript compiler to resolve the url */ + private static _specifierCache: Map< + string, + Map + > = new Map(); + + /** Retrieve a `SourceFile` based on a `moduleSpecifier` and `containingFile` + * or return `undefined` if not preset. */ + static getUrl( + moduleSpecifier: string, + containingFile: string + ): string | undefined { + const containingCache = this._specifierCache.get(containingFile); + if (containingCache) { + const sourceFile = containingCache.get(moduleSpecifier); + return sourceFile && sourceFile.url; + } + return undefined; + } + + /** Retrieve a `SourceFile` based on a `url` */ + static get(url: string): SourceFile | undefined { + return this._moduleCache.get(url); + } + + /** Determine if a source file exists or not */ + static has(url: string): boolean { + return this._moduleCache.has(url); + } +} diff --git a/cli/js/type_directives.ts b/cli/js/compiler_type_directives.ts similarity index 100% rename from cli/js/type_directives.ts rename to cli/js/compiler_type_directives.ts diff --git a/cli/js/compiler_util.ts b/cli/js/compiler_util.ts new file mode 100644 index 00000000000000..30c6f61629fcff --- /dev/null +++ b/cli/js/compiler_util.ts @@ -0,0 +1,298 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +import { bold, cyan, yellow } from "./colors.ts"; +import { CompilerOptions } from "./compiler_api.ts"; +import { buildBundle } from "./compiler_bundler.ts"; +import { ConfigureResponse, Host } from "./compiler_host.ts"; +import { SourceFile } from "./compiler_sourcefile.ts"; +import { sendSync } from "./dispatch_json.ts"; +import * as dispatch from "./dispatch.ts"; +import { TextEncoder } from "./text_encoding.ts"; +import * as util from "./util.ts"; +import { assert } from "./util.ts"; +import { writeFileSync } from "./write_file.ts"; + +/** Type for the write fall callback that allows delegation from the compiler + * host on writing files. */ +export type WriteFileCallback = ( + fileName: string, + data: string, + sourceFiles?: readonly ts.SourceFile[] +) => void; + +/** An object which is passed to `createWriteFile` to be used to read and set + * state related to the emit of a program. */ +export interface WriteFileState { + type: CompilerRequestType; + bundle?: boolean; + host?: Host; + outFile?: string; + rootNames: string[]; + emitMap?: Record; + emitBundle?: string; + sources?: Record; +} + +// Warning! The values in this enum are duplicated in `cli/msg.rs` +// Update carefully! +export enum CompilerRequestType { + Compile = 0, + RuntimeCompile = 1, + RuntimeTranspile = 2 +} + +export const OUT_DIR = "$deno$"; + +/** Cache the contents of a file on the trusted side. */ +function cache( + moduleId: string, + emittedFileName: string, + contents: string, + checkJs = false +): void { + util.log("compiler::cache", { moduleId, emittedFileName, checkJs }); + const sf = SourceFile.get(moduleId); + + if (sf) { + // NOTE: If it's a `.json` file we don't want to write it to disk. + // JSON files are loaded and used by TS compiler to check types, but we don't want + // to emit them to disk because output file is the same as input file. + if (sf.extension === ts.Extension.Json) { + return; + } + + // NOTE: JavaScript files are only cached to disk if `checkJs` + // option in on + if (sf.extension === ts.Extension.Js && !checkJs) { + return; + } + } + + if (emittedFileName.endsWith(".map")) { + // Source Map + sendSync(dispatch.OP_CACHE, { + extension: ".map", + moduleId, + contents + }); + } else if ( + emittedFileName.endsWith(".js") || + emittedFileName.endsWith(".json") + ) { + // Compiled JavaScript + sendSync(dispatch.OP_CACHE, { + extension: ".js", + moduleId, + contents + }); + } else { + assert(false, `Trying to cache unhandled file type "${emittedFileName}"`); + } +} + +const encoder = new TextEncoder(); + +/** Generates a `writeFile` function which can be passed to the compiler `Host` + * to use when emitting files. */ +export function createWriteFile(state: WriteFileState): WriteFileCallback { + if (state.type === CompilerRequestType.Compile) { + return function writeFile( + fileName: string, + data: string, + sourceFiles?: readonly ts.SourceFile[] + ): void { + assert( + sourceFiles != null, + `Unexpected emit of "${fileName}" which isn't part of a program.` + ); + assert(state.host); + if (!state.bundle) { + assert(sourceFiles.length === 1); + cache( + sourceFiles[0].fileName, + fileName, + data, + state.host.getCompilationSettings().checkJs + ); + } else { + // if the fileName is set to an internal value, just noop, this is + // used in the Rust unit tests. + if (state.outFile && state.outFile.startsWith(OUT_DIR)) { + return; + } + // we only support single root names for bundles + assert( + state.rootNames.length === 1, + `Only one root name supported. Got "${JSON.stringify( + state.rootNames + )}"` + ); + // this enriches the string with the loader and re-exports the + // exports of the root module + const content = buildBundle(state.rootNames[0], data, sourceFiles); + if (state.outFile) { + const encodedData = encoder.encode(content); + console.warn(`Emitting bundle to "${state.outFile}"`); + writeFileSync(state.outFile, encodedData); + console.warn(`${util.humanFileSize(encodedData.length)} emitted.`); + } else { + console.log(content); + } + } + }; + } + + return function writeFile( + fileName: string, + data: string, + sourceFiles?: readonly ts.SourceFile[] + ): void { + assert(sourceFiles != null); + assert(state.host); + assert(state.emitMap); + if (!state.bundle) { + assert(sourceFiles.length === 1); + state.emitMap[fileName] = data; + // we only want to cache the compiler output if we are resolving + // modules externally + if (!state.sources) { + cache( + sourceFiles[0].fileName, + fileName, + data, + state.host.getCompilationSettings().checkJs + ); + } + } else { + // we only support single root names for bundles + assert(state.rootNames.length === 1); + state.emitBundle = buildBundle(state.rootNames[0], data, sourceFiles); + } + }; +} + +/** Take a runtime set of compiler options as stringified JSON and convert it + * to a set of TypeScript compiler options. */ +export function convertCompilerOptions(str: string): ts.CompilerOptions { + const options: CompilerOptions = JSON.parse(str); + const out: Record = {}; + const keys = Object.keys(options) as Array; + for (const key of keys) { + switch (key) { + case "jsx": + const value = options[key]; + if (value === "preserve") { + out[key] = ts.JsxEmit.Preserve; + } else if (value === "react") { + out[key] = ts.JsxEmit.React; + } else { + out[key] = ts.JsxEmit.ReactNative; + } + break; + case "module": + switch (options[key]) { + case "amd": + out[key] = ts.ModuleKind.AMD; + break; + case "commonjs": + out[key] = ts.ModuleKind.CommonJS; + break; + case "es2015": + case "es6": + out[key] = ts.ModuleKind.ES2015; + break; + case "esnext": + out[key] = ts.ModuleKind.ESNext; + break; + case "none": + out[key] = ts.ModuleKind.None; + break; + case "system": + out[key] = ts.ModuleKind.System; + break; + case "umd": + out[key] = ts.ModuleKind.UMD; + break; + default: + throw new TypeError("Unexpected module type"); + } + break; + case "target": + switch (options[key]) { + case "es3": + out[key] = ts.ScriptTarget.ES3; + break; + case "es5": + out[key] = ts.ScriptTarget.ES5; + break; + case "es6": + case "es2015": + out[key] = ts.ScriptTarget.ES2015; + break; + case "es2016": + out[key] = ts.ScriptTarget.ES2016; + break; + case "es2017": + out[key] = ts.ScriptTarget.ES2017; + break; + case "es2018": + out[key] = ts.ScriptTarget.ES2018; + break; + case "es2019": + out[key] = ts.ScriptTarget.ES2019; + break; + case "es2020": + out[key] = ts.ScriptTarget.ES2020; + break; + case "esnext": + out[key] = ts.ScriptTarget.ESNext; + break; + default: + throw new TypeError("Unexpected emit target."); + } + default: + out[key] = options[key]; + } + } + return out as ts.CompilerOptions; +} + +/** An array of TypeScript diagnostic types we ignore. */ +export const ignoredDiagnostics = [ + // TS1103: 'for-await-of' statement is only allowed within an async function + // or async generator. + 1103, + // TS1308: 'await' expression is only allowed within an async function. + 1308, + // TS2691: An import path cannot end with a '.ts' extension. Consider + // importing 'bad-module' instead. + 2691, + // TS5009: Cannot find the common subdirectory path for the input files. + 5009, + // TS5055: Cannot write file + // 'http://localhost:4545/tests/subdir/mt_application_x_javascript.j4.js' + // because it would overwrite input file. + 5055, + // TypeScript is overly opinionated that only CommonJS modules kinds can + // support JSON imports. Allegedly this was fixed in + // Microsoft/TypeScript#26825 but that doesn't seem to be working here, + // so we will ignore complaints about this compiler setting. + 5070 +]; + +/** When doing a host configuration, processing the response and logging out + * and options which were ignored. */ +export function processConfigureResponse( + configResult: ConfigureResponse, + configPath: string +): ts.Diagnostic[] | undefined { + const { ignoredOptions, diagnostics } = configResult; + if (ignoredOptions) { + console.warn( + yellow(`Unsupported compiler options in "${configPath}"\n`) + + cyan(` The following options were ignored:\n`) + + ` ${ignoredOptions.map((value): string => bold(value)).join(", ")}` + ); + } + return diagnostics; +} diff --git a/cli/js/globals.ts b/cli/js/globals.ts index 3027d60aa9504b..68cdaec4a65b46 100644 --- a/cli/js/globals.ts +++ b/cli/js/globals.ts @@ -62,6 +62,8 @@ declare global { interface Object { [consoleTypes.customInspect]?(): string; } + + const console: consoleTypes.Console; } // A self reference to the global object. diff --git a/cli/tests/compiler_api_test.ts b/cli/tests/compiler_api_test.ts index c7af466fb5f54e..7cb5029f6f58df 100644 --- a/cli/tests/compiler_api_test.ts +++ b/cli/tests/compiler_api_test.ts @@ -1,6 +1,7 @@ -const result = await Deno.bundle("/foo/index.ts", { - "/foo/index.ts": `import * as bar from "../bar.ts";\n\nconsole.log(bar);\n`, + +const [diagnostics, actual] = await Deno.compile("/foo.ts", { + "/foo.ts": `import * as bar from "./bar.ts";\n\nconsole.log(bar);\n`, "/bar.ts": `export const bar = "bar";\n` }); -console.log(JSON.stringify(result, undefined, " ")); +console.log(diagnostics, actual); diff --git a/cli/tests/error_011_bad_module_specifier.ts.out b/cli/tests/error_011_bad_module_specifier.ts.out index 5f1418c26a8d38..7c100db13ad2cf 100644 --- a/cli/tests/error_011_bad_module_specifier.ts.out +++ b/cli/tests/error_011_bad_module_specifier.ts.out @@ -3,6 +3,6 @@ at DenoError ($deno$/errors.ts:[WILDCARD]) at unwrapResponse ($deno$/dispatch_json.ts:[WILDCARD]) at sendSync ($deno$/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) + at resolveModules ($deno$/compiler_imports.ts:[WILDCARD]) + at processImports ($deno$/compiler_imports.ts:[WILDCARD]) + at processImports ($deno$/compiler_imports.ts:[WILDCARD]) diff --git a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out index bdd08245128db9..095ca497bfc2c6 100644 --- a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out +++ b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out @@ -3,6 +3,6 @@ at DenoError ($deno$/errors.ts:[WILDCARD]) at unwrapResponse ($deno$/dispatch_json.ts:[WILDCARD]) at sendSync ($deno$/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) + at resolveModules ($deno$/compiler_imports.ts:[WILDCARD]) + at processImports ($deno$/compiler_imports.ts:[WILDCARD]) + at processImports ($deno$/compiler_imports.ts:[WILDCARD]) diff --git a/cli/tests/error_type_definitions.ts.out b/cli/tests/error_type_definitions.ts.out index f613a7e2f66619..d2c6096aced33c 100644 --- a/cli/tests/error_type_definitions.ts.out +++ b/cli/tests/error_type_definitions.ts.out @@ -3,6 +3,6 @@ at DenoError ($deno$/errors.ts:[WILDCARD]) at unwrapResponse ($deno$/dispatch_json.ts:[WILDCARD]) at sendSync ($deno$/dispatch_json.ts:[WILDCARD]) - at resolveModules ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) - at processImports ($deno$/compiler.ts:[WILDCARD]) + at resolveModules ($deno$/compiler_imports.ts:[WILDCARD]) + at processImports ($deno$/compiler_imports.ts:[WILDCARD]) + at processImports ($deno$/compiler_imports.ts:[WILDCARD])