Compile TypeScript incrementally

Author: mzdunek93

cli/compilers/ts.rs

@@ -254,6 +254,7 @@ struct BundleResponse {
 struct CompileResponse {
   diagnostics: Diagnostic,
   emit_map: HashMap<String, EmittedSource>,
+  sources: Vec<String>,
 }
 
 // TODO(bartlomieju): possible deduplicate once TS refactor is stabilized
@@ -392,29 +393,6 @@ impl TsCompiler {
       return self.get_compiled_module(&source_file.url);
     }
 
-    if self.use_disk_cache {
-      // Try to load cached version:
-      // 1. check if there's 'meta' file
-      if let Some(metadata) = self.get_metadata(&source_file.url) {
-        // 2. compare version hashes
-        // TODO: it would probably be good idea to make it method implemented on SourceFile
-        let version_hash_to_validate = source_code_version_hash(
-          &source_file.source_code,
-          version::DENO,
-          &self.config.hash,
-        );
-
-        if metadata.version_hash == version_hash_to_validate {
-          debug!("load_cache metadata version hash match");
-          if let Ok(compiled_module) =
-            self.get_compiled_module(&source_file.url)
-          {
-            self.mark_compiled(&source_file.url);
-            return Ok(compiled_module);
-          }
-        }
-      }
-    }
     let source_file_ = source_file.clone();
     let module_url = source_file.url.clone();
     let target = match target {
@@ -444,6 +422,11 @@ impl TsCompiler {
 
     let compile_response: CompileResponse = serde_json::from_str(json_str)?;
 
+    for source in compile_response.sources.iter() {
+      let ref url = Url::parse(source)?;
+      self.mark_compiled(url);
+    }
+
     if !compile_response.diagnostics.items.is_empty() {
       return Err(ErrBox::from(compile_response.diagnostics));
     }
@@ -484,6 +467,8 @@ impl TsCompiler {
         self.cache_source_map(&specifier, &source.contents)?;
       } else if emitted_name.ends_with(".js") {
         self.cache_compiled_file(&specifier, &source.contents)?;
+      } else if emitted_name.ends_with("tsbuildinfo.json") {
+        self.cache_build_info(&specifier, &source.contents)?;
       } else {
         panic!("Trying to cache unknown file type {}", emitted_name);
       }
@@ -558,7 +543,6 @@ impl TsCompiler {
       .disk_cache
       .get_cache_filename_with_extension(module_specifier.as_url(), "js");
     self.disk_cache.set(&js_key, contents.as_bytes())?;
-    self.mark_compiled(module_specifier.as_url());
 
     let version_hash = source_code_version_hash(
       &source_file.source_code,
@@ -627,6 +611,18 @@ impl TsCompiler {
       .get_cache_filename_with_extension(module_specifier.as_url(), "js.map");
     self.disk_cache.set(&source_map_key, contents.as_bytes())
   }
+
+  /// Save TS build info file to on-disk cache.
+  fn cache_build_info(
+    &self,
+    module_specifier: &ModuleSpecifier,
+    contents: &str,
+  ) -> std::io::Result<()> {
+    let build_info_key = self
+      .disk_cache
+      .get_cache_filename(module_specifier.as_url());
+    self.disk_cache.set(&build_info_key, contents.as_bytes())
+  }
 }
 
 impl SourceMapGetter for TsCompiler {

cli/js/compiler.ts

@@ -45,6 +45,7 @@ import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts";
 import { assert } from "./util.ts";
 import * as util from "./util.ts";
 import { bootstrapWorkerRuntime } from "./runtime_worker.ts";
+import { SourceFile } from "./compiler/sourcefile.ts";
 
 interface CompilerRequestCompile {
   type: CompilerRequestType.Compile;
@@ -83,6 +84,7 @@ type CompilerRequest =
 interface CompileResult {
   emitMap?: Record<string, EmmitedSource>;
   bundleOutput?: string;
+  sources: string[],
   diagnostics: Diagnostic;
 }
 
@@ -131,6 +133,7 @@ async function compile(
     writeFile = createCompileWriteFile(state);
   }
   const host = (state.host = new Host({
+    rootNames,
     bundle,
     target,
     writeFile,
@@ -157,8 +160,9 @@ async function compile(
   // to generate the program and possibly emit it.
   if (diagnostics.length === 0) {
     const options = host.getCompilationSettings();
+    const relativeRootNames = rootNames.map(host.getRelativePath.bind(host))
     const program = ts.createProgram({
-      rootNames,
+      rootNames: relativeRootNames,
       options,
       host,
       oldProgram: TS_SNAPSHOT_PROGRAM,
@@ -175,11 +179,15 @@ async function compile(
         assert(resolvedRootModules.length === 1);
         setRootExports(program, resolvedRootModules[0]);
       }
-      const emitResult = program.emit();
-      assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
+      const reportDiagnostic: ts.DiagnosticReporter = (diagnostic) =>
+        diagnostics = !ignoredDiagnostics.includes(diagnostic.code) ? [...(diagnostics ?? []), diagnostic] : diagnostics;
+      // @ts-ignore
+      const exitStatus = ts.performIncrementalCompilation({ rootNames: relativeRootNames, options, host, reportDiagnostic });
+      const emitSkipped = exitStatus === ts.ExitStatus.DiagnosticsPresent_OutputsSkipped;
+
+      assert(emitSkipped === false, "Unexpected skip of the emit.");      
       // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned
       // without casting.
-      diagnostics = emitResult.diagnostics;
     }
   }
 
@@ -194,6 +202,7 @@ async function compile(
   const result: CompileResult = {
     emitMap: state.emitMap,
     bundleOutput,
+    sources: SourceFile.urls(),
     diagnostics: fromTypeScriptDiagnostic(diagnostics),
   };
 

cli/js/compiler/host.ts

@@ -1,7 +1,7 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
 import { ASSETS, MediaType, SourceFile } from "./sourcefile.ts";
-import { OUT_DIR, WriteFileCallback, getAsset } from "./util.ts";
+import { OUT_DIR, WriteFileCallback, getAsset, getCache, TS_BUILD_INFO } from "./util.ts";
 import { assert, notImplemented } from "../util.ts";
 import * as util from "../util.ts";
 
@@ -16,6 +16,7 @@ export interface CompilerHostOptions {
   target: CompilerHostTarget;
   unstable?: boolean;
   writeFile: WriteFileCallback;
+  rootNames?: string[];
 }
 
 export interface ConfigureResponse {
@@ -38,14 +39,17 @@ export const defaultCompileOptions: ts.CompilerOptions = {
   allowNonTsExtensions: true,
   checkJs: false,
   esModuleInterop: true,
+  incremental: true,
   jsx: ts.JsxEmit.React,
   module: ts.ModuleKind.ESNext,
+  moduleResolution: ts.ModuleResolutionKind.NodeJs,
   outDir: OUT_DIR,
   resolveJsonModule: true,
   sourceMap: true,
   strict: true,
   stripComments: true,
   target: ts.ScriptTarget.ESNext,
+  tsBuildInfoFile: TS_BUILD_INFO,
 };
 
 export const defaultRuntimeCompileOptions: ts.CompilerOptions = {
@@ -141,6 +145,8 @@ export class Host implements ts.CompilerHost {
   readonly #options = defaultCompileOptions;
   #target: CompilerHostTarget;
   #writeFile: WriteFileCallback;
+  #rootPath = '.';
+  #rootName = '';
 
   /* Deno specific APIs */
 
@@ -149,6 +155,7 @@ export class Host implements ts.CompilerHost {
     target,
     unstable,
     writeFile,
+    rootNames
   }: CompilerHostOptions) {
     this.#target = target;
     this.#writeFile = writeFile;
@@ -164,6 +171,10 @@ export class Host implements ts.CompilerHost {
         "lib.deno.unstable.d.ts",
       ];
     }
+    if (rootNames) {
+      this.#rootName = rootNames[0];
+      this.#rootPath = this.#rootName.split('/').slice(0, -1).join('/');
+    }
   }
 
   configure(
@@ -240,6 +251,25 @@ export class Host implements ts.CompilerHost {
     return "\n";
   }
 
+  getAbsolutePath(fileName: string): string {
+    //@ts-ignore
+    return ts.resolvePath(this.#rootPath, fileName);
+  }
+
+  getModuleAbsolutePath(containingFile: string, specifier: string) {
+    //@ts-ignore
+    return ts.resolvePath(ts.getDirectoryPath(this.getAbsolutePath(containingFile)), specifier);
+  }
+
+  getRelativePath(fileName: string): string {
+    //@ts-ignore
+    if (!ts.pathIsAbsolute(fileName)) {
+      return fileName;
+    }
+    //@ts-ignore
+    return ts.getRelativePathFromDirectory(this.#rootPath, fileName);
+  }
+
   getSourceFile(
     fileName: string,
     languageVersion: ts.ScriptTarget,
@@ -251,21 +281,24 @@ export class Host implements ts.CompilerHost {
       assert(!shouldCreateNewSourceFile);
       const sourceFile = fileName.startsWith(ASSETS)
         ? getAssetInternal(fileName)
-        : SourceFile.get(fileName);
+        : SourceFile.get(this.getAbsolutePath(fileName));
       assert(sourceFile != null);
       if (!sourceFile.tsSourceFile) {
         assert(sourceFile.sourceCode != null);
         const tsSourceFileName = fileName.startsWith(ASSETS)
           ? sourceFile.filename
-          : fileName;
+          : this.getRelativePath(fileName);
 
         sourceFile.tsSourceFile = ts.createSourceFile(
           tsSourceFileName,
           sourceFile.sourceCode,
           languageVersion
         );
+        //@ts-ignore
+        sourceFile.tsSourceFile.version = this.createHash(sourceFile.tsSourceFile.text);
         delete sourceFile.sourceCode;
       }
+
       return sourceFile.tsSourceFile;
     } catch (e) {
       if (onError) {
@@ -277,8 +310,17 @@ export class Host implements ts.CompilerHost {
     }
   }
 
-  readFile(_fileName: string): string | undefined {
-    return notImplemented();
+  readFile(fileName: string): string | undefined {
+    util.log("compiler::host.readFile", fileName);
+    if (fileName === TS_BUILD_INFO) {
+      return getCache(`${this.#rootName}.json`);
+    }
+    return getCache(this.getAbsolutePath(fileName.replace(OUT_DIR, this.#rootPath)));
+  }
+
+  createHash(data: string): string {
+    //@ts-ignore
+    return ts.generateDjb2Hash(data);
   }
 
   resolveModuleNames(
@@ -290,22 +332,23 @@ export class Host implements ts.CompilerHost {
       containingFile,
     });
     return moduleNames.map((specifier) => {
-      const maybeUrl = SourceFile.getUrl(specifier, containingFile);
+      const url = this.getModuleAbsolutePath(containingFile, specifier);
 
       let sourceFile: SourceFile | undefined = undefined;
 
       if (specifier.startsWith(ASSETS)) {
         sourceFile = getAssetInternal(specifier);
-      } else if (typeof maybeUrl !== "undefined") {
-        sourceFile = SourceFile.get(maybeUrl);
+      } else {
+        sourceFile = SourceFile.get(url);
       }
 
       if (!sourceFile) {
         return undefined;
       }
 
+      const resolvedFileName = specifier.startsWith(ASSETS) ? sourceFile.url : this.getRelativePath(sourceFile.url);
       return {
-        resolvedFileName: sourceFile.url,
+        resolvedFileName,
         isExternalLibraryImport: specifier.startsWith(ASSETS),
         extension: sourceFile.extension,
       };
@@ -324,6 +367,6 @@ export class Host implements ts.CompilerHost {
     sourceFiles?: readonly ts.SourceFile[]
   ): void {
     util.log("compiler::host.writeFile", fileName);
-    this.#writeFile(fileName, data, sourceFiles);
+    this.#writeFile(this.getAbsolutePath(fileName.replace(OUT_DIR, this.#rootPath)), data, sourceFiles);
   }
 }

cli/js/compiler/sourcefile.ts

@@ -162,4 +162,9 @@ export class SourceFile {
   static get(url: string): SourceFile | undefined {
     return moduleCache.get(url);
   }
+
+  static urls(): string[] {
+    //@ts-ignore
+    return Array.from(moduleCache.keys()).filter(ts.pathIsAbsolute);
+  }
 }

cli/js/compiler/util.ts

@@ -40,11 +40,17 @@ export enum CompilerRequestType {
 }
 
 export const OUT_DIR = "$deno$";
+export const TS_BUILD_INFO = `${OUT_DIR}/tsbuildinfo.json`;
 
 export function getAsset(name: string): string {
   return compilerOps.getAsset(name);
 }
 
+export function getCache(name: string): string | undefined {
+  const { content } = compilerOps.getCache(name);
+  return content;
+}
+
 // TODO(bartlomieju): probably could be defined inline?
 export function createBundleWriteFile(
   state: WriteFileState
@@ -68,18 +74,25 @@ export function createBundleWriteFile(
 export function createCompileWriteFile(
   state: WriteFileState
 ): WriteFileCallback {
+  const rootPath = state.rootNames[0].split("/").slice(0, -1).join("/");
   return function writeFile(
     fileName: string,
     data: string,
     sourceFiles?: readonly ts.SourceFile[]
   ): void {
-    assert(sourceFiles != null);
+    const isBuildInfo = fileName === TS_BUILD_INFO.replace(OUT_DIR, rootPath);
+    if (sourceFiles != null) {
+      assert(sourceFiles.length === 1);
+    } else {
+      assert(isBuildInfo);
+    }
     assert(state.host);
     assert(state.emitMap);
     assert(!state.bundle);
-    assert(sourceFiles.length === 1);
+    //@ts-ignore
+    const filename = isBuildInfo ? `${state.rootNames[0]}.json` : ts.resolvePath(rootPath, sourceFiles[0].fileName);
     state.emitMap[fileName] = {
-      filename: sourceFiles[0].fileName,
+      filename,
       contents: data,
     };
   };

cli/js/ops/compiler.ts

@@ -38,3 +38,9 @@ export function getAsset(name: string): string {
   const sourceCodeBytes = core.dispatch(opId, encoder.encode(name));
   return decoder.decode(sourceCodeBytes!);
 }
+
+export function getCache(
+  url: string
+): { content?: string } {
+  return sendSync("op_get_cache", { url });
+}