Skip to content

Commit

Permalink
perf(compiler): don’t write compile output to file system but rely on…
Browse files Browse the repository at this point in the history
… jest cache (#1561)
  • Loading branch information
ahnpnl committed May 4, 2020
1 parent 02722ee commit d11a4ea
Show file tree
Hide file tree
Showing 10 changed files with 68 additions and 249 deletions.
30 changes: 15 additions & 15 deletions e2e/__tests__/__snapshots__/logger.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,19 @@ Array [
"[level:20] processing <cwd>/Hello.spec.ts",
"[level:20] file caching disabled",
"[level:20] initializeLanguageServiceInstance(): create typescript compiler",
"[level:20] compileUsingLanguageService(): creating language service",
"[level:20] compileAndCacheResult(): no cache",
"[level:20] initializeLanguageServiceInstance(): creating language service",
"[level:20] compileAndCacheResult(): get compile output",
"[level:20] compileFn(): compiling using language service",
"[level:20] updateMemoryCache(): update memory cache for language service",
"[level:20] visitSourceFileNode(): hoisting",
"[level:20] compileFn(): computing diagnostics for <cwd>/Hello.spec.ts using language service",
"[level:20] compileFn(): computing diagnostics using language service",
"[level:20] computing cache key for <cwd>/Hello.ts",
"[level:20] processing <cwd>/Hello.ts",
"[level:20] compileAndCacheResult(): no cache",
"[level:20] compileAndCacheResult(): get compile output",
"[level:20] compileFn(): compiling using language service",
"[level:20] updateMemoryCache(): update memory cache for language service",
"[level:20] visitSourceFileNode(): hoisting",
"[level:20] compileFn(): computing diagnostics for <cwd>/Hello.ts using language service",
"[level:20] compileFn(): computing diagnostics using language service",
]
`;
Expand Down Expand Up @@ -62,20 +62,20 @@ Array [
"[level:20] checking version of babel-jest: OK",
"[level:20] file caching disabled",
"[level:20] initializeLanguageServiceInstance(): create typescript compiler",
"[level:20] compileUsingLanguageService(): creating language service",
"[level:20] compileAndCacheResult(): no cache",
"[level:20] initializeLanguageServiceInstance(): creating language service",
"[level:20] compileAndCacheResult(): get compile output",
"[level:20] compileFn(): compiling using language service",
"[level:20] updateMemoryCache(): update memory cache for language service",
"[level:20] visitSourceFileNode(): hoisting",
"[level:20] compileFn(): computing diagnostics for <cwd>/Hello.spec.ts using language service",
"[level:20] compileFn(): computing diagnostics using language service",
"[level:20] calling babel-jest processor",
"[level:20] computing cache key for <cwd>/Hello.ts",
"[level:20] processing <cwd>/Hello.ts",
"[level:20] compileAndCacheResult(): no cache",
"[level:20] compileAndCacheResult(): get compile output",
"[level:20] compileFn(): compiling using language service",
"[level:20] updateMemoryCache(): update memory cache for language service",
"[level:20] visitSourceFileNode(): hoisting",
"[level:20] compileFn(): computing diagnostics for <cwd>/Hello.ts using language service",
"[level:20] compileFn(): computing diagnostics using language service",
"[level:20] calling babel-jest processor",
]
`;
Expand Down Expand Up @@ -106,20 +106,20 @@ Array [
"[level:20] checking version of babel-jest: OK",
"[level:20] file caching disabled",
"[level:20] initializeLanguageServiceInstance(): create typescript compiler",
"[level:20] compileUsingLanguageService(): creating language service",
"[level:20] compileAndCacheResult(): no cache",
"[level:20] initializeLanguageServiceInstance(): creating language service",
"[level:20] compileAndCacheResult(): get compile output",
"[level:20] compileFn(): compiling using language service",
"[level:20] updateMemoryCache(): update memory cache for language service",
"[level:20] visitSourceFileNode(): hoisting",
"[level:20] compileFn(): computing diagnostics for <cwd>/Hello.spec.ts using language service",
"[level:20] compileFn(): computing diagnostics using language service",
"[level:20] calling babel-jest processor",
"[level:20] computing cache key for <cwd>/Hello.ts",
"[level:20] processing <cwd>/Hello.ts",
"[level:20] compileAndCacheResult(): no cache",
"[level:20] compileAndCacheResult(): get compile output",
"[level:20] compileFn(): compiling using language service",
"[level:20] updateMemoryCache(): update memory cache for language service",
"[level:20] visitSourceFileNode(): hoisting",
"[level:20] compileFn(): computing diagnostics for <cwd>/Hello.ts using language service",
"[level:20] compileFn(): computing diagnostics using language service",
"[level:20] calling babel-jest processor",
]
`;
Expand Down
16 changes: 0 additions & 16 deletions src/compiler/__snapshots__/language-service.spec.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -90,19 +90,3 @@ exports[`Language service should throw error when cannot compile 1`] = `
"Unable to require \`.d.ts\` file for file: test-cannot-compile.d.ts.
This is usually the result of a faulty configuration or import. Make sure there is a \`.js\`, \`.json\` or another executable extension available alongside \`test-cannot-compile.d.ts\`."
`;

exports[`Language service should use the cache 3`] = `
===[ FILE: test-cache.ts ]======================================================
console.log("hello");
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJmaWxlIjoidGVzdC1jYWNoZS50cyIsIm1hcHBpbmdzIjoiQUFBQSxPQUFPLENBQUMsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFBIiwibmFtZXMiOltdLCJzb3VyY2VzIjpbInRlc3QtY2FjaGUudHMiXSwic291cmNlc0NvbnRlbnQiOlsiY29uc29sZS5sb2coXCJoZWxsb1wiKSJdLCJ2ZXJzaW9uIjozfQ==
===[ INLINE SOURCE MAPS ]=======================================================
file: test-cache.ts
mappings: 'AAAA,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA'
names: []
sources:
- test-cache.ts
sourcesContent:
- console.log("hello")
version: 3
================================================================================
`;
18 changes: 0 additions & 18 deletions src/compiler/__snapshots__/transpiler.spec.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -66,24 +66,6 @@ exports[`Transpiler should compile tsx file for other jsx options 1`] = `
================================================================================
`;
exports[`Transpiler should compile using transpileModule and not use cache 1`] = `
===[ FILE: src/compiler/transpiler.spec.ts ]====================================
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = 42;
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJmaWxlIjoiPGN3ZD4vc3JjL2NvbXBpbGVyL3RyYW5zcGlsZXIuc3BlYy50cyIsIm1hcHBpbmdzIjoiOztBQUFBLGtCQUFlLEVBQUUsQ0FBQSIsIm5hbWVzIjpbXSwic291cmNlcyI6WyI8Y3dkPi9zcmMvY29tcGlsZXIvdHJhbnNwaWxlci5zcGVjLnRzIl0sInNvdXJjZXNDb250ZW50IjpbImV4cG9ydCBkZWZhdWx0IDQyIl0sInZlcnNpb24iOjN9
===[ INLINE SOURCE MAPS ]=======================================================
file: <cwd>/src/compiler/transpiler.spec.ts
mappings: ';;AAAA,kBAAe,EAAE,CAAA'
names: []
sources:
- <cwd>/src/compiler/transpiler.spec.ts
sourcesContent:
- export default 42
version: 3
================================================================================
`;
exports[`Transpiler should report diagnostics related to codes with pathRegex config is undefined 1`] = `"foo.ts(2,23): error TS1005: '=>' expected."`;
exports[`Transpiler should report diagnostics related to codes with pathRegex config matches file name 1`] = `"foo.ts(2,23): error TS1005: '=>' expected."`;
12 changes: 8 additions & 4 deletions src/compiler/compiler-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ import { EXTENSION_REGEX, JSON_REGEX, TS_TSX_REGEX } from '../constants'
import { MemoryCache, SourceOutput, TSFiles } from '../types'
import { sha1 } from '../util/sha1'

/**
* @internal
*/
export const hasOwn = Object.prototype.hasOwnProperty
/**
* @internal
*/
Expand All @@ -35,7 +39,7 @@ export function cacheResolvedModules(
* Ugly trick while waiting for https://github.com/microsoft/TypeScript/issues/33994
*/
if (importReferences.length) {
logger.debug({ fileName }, `cacheResolvedModules(): get resolved modules of test file ${fileName}`)
logger.debug({ fileName }, `cacheResolvedModules(): get resolved modules`)

memoryCache.resolvedModules[fileName] = Object.create(null)
memoryCache.resolvedModules[fileName].modulePaths = importReferences
Expand Down Expand Up @@ -113,7 +117,7 @@ export function getAndCacheProjectReference(
files: TSFiles,
projectReferences: ReadonlyArray<_ts.ProjectReference> | undefined,
) {
const file = files.get(filePath)
const file = files[filePath]
if (file !== undefined && file.projectReference) {
return file.projectReference.project
}
Expand Down Expand Up @@ -152,8 +156,8 @@ function getAndCacheOutputJSFileName(
projectReference: _ts.ResolvedProjectReference,
files: TSFiles,
) {
const file = files.get(inputFileName)
if (file && file.projectReference && file.projectReference.outputFileName) {
const file = files[inputFileName]
if (file?.projectReference && file.projectReference.outputFileName) {
return file.projectReference.outputFileName
}

Expand Down
111 changes: 16 additions & 95 deletions src/compiler/instance.ts
Original file line number Diff line number Diff line change
@@ -1,42 +1,10 @@
/**
* This code is heavily inspired from
* https://github.com/JsCommunity/make-error/blob/v1.3.4/index.js
* ...but more modified than expected :-D
* Below is the original license anyway:
*
* ---
*
* The MIT License (MIT)
*
* Copyright (c) 2014 Blake Embrey ([email protected])
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/

import { Logger } from 'bs-logger'
import { readFileSync, writeFileSync } from 'fs'
import { readFileSync } from 'fs'
import mkdirp = require('mkdirp')
import { basename, extname, join } from 'path'
import { basename, extname } from 'path'

import { ConfigSet } from '../config/config-set'
import { CompileFn, CompilerInstance, MemoryCache, TSFile, TsCompiler } from '../types'
import { sha1 } from '../util/sha1'
import { CompileFn, CompilerInstance, MemoryCache, TsCompiler } from '../types'

import { getResolvedModulesCache } from './compiler-utils'
import { initializeLanguageServiceInstance } from './language-service'
Expand Down Expand Up @@ -72,74 +40,27 @@ const updateSourceMap = (sourceMapText: string, normalizedFileName: string): str
return JSON.stringify(sourceMap)
}

/**
* Get the file name for the cache entry.
*/
const getCacheName = (sourceCode: string, normalizedFileName: string): string => {
return sha1(normalizedFileName, '\x00', sourceCode)
}

/**
* Ensure the given cached content is valid by sniffing for a base64 encoded '}'
* at the end of the content, which should exist if there is a valid sourceMap present.
*/
const isValidCacheContent = (contents: string): boolean => {
return /(?:9|0=|Q==)$/.test(contents.slice(-3))
}

/**
* Compile files which are provided by jest via transform config and cache the result in file system if users run with
* cache mode
*/
const compileAndCacheResult = (
cacheDir: string | undefined,
memoryCache: MemoryCache,
compileFn: CompileFn,
getExtension: (fileName: string) => string,
logger: Logger,
) => {
return (code: string, fileName: string, lineOffset?: number) => {
function getCompileOutput(): string {
const [value, sourceMap] = compileFn(code, fileName, lineOffset)
const output = updateOutput(value, fileName, sourceMap, getExtension)
memoryCache.files.set(fileName, {
...memoryCache.files.get(fileName)!,
output,
})
logger.debug({ fileName }, 'compileAndCacheResult(): get compile output')

return output
const [value, sourceMap] = compileFn(code, fileName, lineOffset)
const output = updateOutput(value, fileName, sourceMap, getExtension)
memoryCache.files[fileName] = {
...memoryCache.files[fileName],
output,
}
if (!cacheDir) {
logger.debug({ fileName }, 'compileAndCacheResult(): no cache')

return getCompileOutput()
} else {
const cachePath = join(cacheDir, getCacheName(code, fileName))
const extension = getExtension(fileName)
const outputPath = `${cachePath}${extension}`
try {
const output = readFileSync(outputPath, 'utf8')
if (isValidCacheContent(output)) {
logger.debug({ fileName }, 'compileAndCacheResult(): cache hit')
memoryCache.files.set(fileName, {
...memoryCache.files.get(fileName)!,
output,
})

return output
}
} catch (err) {}

logger.debug({ fileName }, 'compileAndCacheResult(): cache miss')

const output = getCompileOutput()

logger.debug({ fileName, outputPath }, 'compileAndCacheResult(): writing caches')

writeFileSync(outputPath, output)

return output
}
return output
}
}

Expand All @@ -157,8 +78,8 @@ export const createCompilerInstance = (configs: ConfigSet): TsCompiler => {
const ts = configs.compilerModule // Require the TypeScript compiler and configuration.
const extensions = ['.ts', '.tsx']
const memoryCache: MemoryCache = {
files: Object.create(null),
resolvedModules: Object.create(null),
files: new Map<string, TSFile>(),
}
// Enable `allowJs` when flag is set.
if (compilerOptions.allowJs) {
Expand All @@ -169,16 +90,16 @@ export const createCompilerInstance = (configs: ConfigSet): TsCompiler => {
// Make sure the cache directory exists before continuing.
mkdirp.sync(cacheDir)
try {
const resolvedModulesCache = readFileSync(getResolvedModulesCache(cacheDir), 'utf-8')
const fsMemoryCache = readFileSync(getResolvedModulesCache(cacheDir), 'utf-8')
/* istanbul ignore next (covered by e2e) */
memoryCache.resolvedModules = JSON.parse(resolvedModulesCache)
memoryCache.resolvedModules = JSON.parse(fsMemoryCache)
} catch (e) {}
}
/* istanbul ignore next (we leave this for e2e) */
configs.jest.setupFiles.concat(configs.jest.setupFilesAfterEnv).forEach(setupFile => {
memoryCache.files.set(setupFile, {
memoryCache.files[setupFile] = {
version: 0,
})
}
})
/**
* Get the extension for a transpiled file.
Expand All @@ -194,7 +115,7 @@ export const createCompilerInstance = (configs: ConfigSet): TsCompiler => {
} else {
compilerInstance = initializeTranspilerInstance(configs, memoryCache, logger)
}
const compile = compileAndCacheResult(cacheDir, memoryCache, compilerInstance.compileFn, getExtension, logger)
const compile = compileAndCacheResult(memoryCache, compilerInstance.compileFn, getExtension, logger)

return { cwd: configs.cwd, compile, program: compilerInstance.program }
}
48 changes: 0 additions & 48 deletions src/compiler/language-service.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { LogLevels } from 'bs-logger'
import { removeSync, writeFileSync } from 'fs-extra'

import { makeCompiler } from '../__helpers__/fakers'
Expand All @@ -16,53 +15,6 @@ describe('Language service', () => {
logTarget.clear()
})

it('should use the cache', () => {
const tmp = tempDir('compiler')
const compiler = makeCompiler({
jestConfig: { cache: true, cacheDirectory: tmp },
tsJestConfig: { tsConfig: false },
})
const source = 'console.log("hello")'
const fileName = 'test-cache.ts'

writeFileSync(fileName, source, 'utf8')

logTarget.clear()
const compiled1 = compiler.compile(source, fileName)

expect(logTarget.filteredLines(LogLevels.debug, Infinity)).toMatchInlineSnapshot(`
Array [
"[level:20] compileAndCacheResult(): cache miss
",
"[level:20] compileFn(): compiling using language service
",
"[level:20] updateMemoryCache(): update memory cache for language service
",
"[level:20] visitSourceFileNode(): hoisting
",
"[level:20] compileFn(): computing diagnostics for test-cache.ts using language service
",
"[level:20] compileAndCacheResult(): writing caches
",
]
`)

logTarget.clear()
const compiled2 = compiler.compile(source, fileName)

expect(logTarget.lines).toMatchInlineSnapshot(`
Array [
"[level:20] compileAndCacheResult(): cache hit
",
]
`)

expect(new ProcessedSource(compiled1, fileName)).toMatchSnapshot()
expect(compiled2).toBe(compiled1)

removeSync(fileName)
})

it('should get compile result from referenced project when there is a built reference project', () => {
const tmp = tempDir('compiler')
const compiler = makeCompiler({
Expand Down
Loading

0 comments on commit d11a4ea

Please sign in to comment.