From 2c971bdecd2602375883851269ef4f71d1e5566c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Jim=C3=A9nez=20Es=C3=BAn?= Date: Wed, 30 May 2018 14:38:58 +0100 Subject: [PATCH] Make jest-haste-map compute SHA-1s for excluded files too (as watchman does) (#6264) --- CHANGELOG.md | 1 + e2e/__tests__/haste_map_sha1.test.js | 81 ++++++++++++++ .../src/__tests__/worker.test.js | 24 ++++- packages/jest-haste-map/src/index.js | 101 ++++++++++-------- packages/jest-haste-map/src/worker.js | 25 ++++- 5 files changed, 180 insertions(+), 52 deletions(-) create mode 100644 e2e/__tests__/haste_map_sha1.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index e4ed1c2d8212..fa8cf7c3e104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * `[expect]` toMatchObject throws TypeError when a source property is null ([#6313](https://github.com/facebook/jest/pull/6313)) * `[jest-cli]` Normalize slashes in paths in CLI output on Windows ([#6310](https://github.com/facebook/jest/pull/6310)) +* `[jest-haste-map`] Compute SHA-1s for non-tracked files when using Node crawler ([#6264](https://github.com/facebook/jest/pull/6264)) ### Chore & Maintenance diff --git a/e2e/__tests__/haste_map_sha1.test.js b/e2e/__tests__/haste_map_sha1.test.js new file mode 100644 index 000000000000..e7f92c54d66f --- /dev/null +++ b/e2e/__tests__/haste_map_sha1.test.js @@ -0,0 +1,81 @@ +/** + * Copyright (c) 2014-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ + +'use strict'; + +import os from 'os'; +import path from 'path'; +import JestHasteMap from 'jest-haste-map'; +const {cleanup, writeFiles} = require('../Utils'); + +const DIR = path.resolve(os.tmpdir(), 'haste_map_sha1'); + +beforeEach(() => cleanup(DIR)); +afterEach(() => cleanup(DIR)); + +test('exits the process after test are done but before timers complete', async () => { + writeFiles(DIR, { + 'file.android.js': '"foo android"', + 'file.ios.js': '"foo ios"', + 'file.js': '"foo default"', + 'file_with_extension.ignored': '"ignored file"', + 'node_modules/bar/file_with_extension.ignored': '"ignored node modules"', + 'node_modules/bar/image.png': '"an image"', + 'node_modules/bar/index.js': '"node modules bar"', + }); + + const haste = new JestHasteMap({ + computeSha1: true, + extensions: ['js', 'json', 'png'], + forceNodeFilesystemAPI: true, + ignorePattern: / ^/, + maxWorkers: 2, + mocksPattern: '', + name: 'tmp', + platforms: ['ios', 'android'], + retainAllFiles: true, + roots: [DIR], + useWatchman: false, + watch: false, + }); + + const {hasteFS} = await haste.build(); + + expect(hasteFS.getSha1(path.join(DIR, 'file.android.js'))).toBe( + 'e376f9fd9a96d000fa019020159f996a8855f8bc', + ); + + expect(hasteFS.getSha1(path.join(DIR, 'file.ios.js'))).toBe( + '1271b4db2a5f47ae46cb01a1d0604a94d401e8f7', + ); + + expect(hasteFS.getSha1(path.join(DIR, 'file.js'))).toBe( + 'c26c852220977244418f17a9fdc4ae9c192b3188', + ); + + expect(hasteFS.getSha1(path.join(DIR, 'node_modules/bar/image.png'))).toBe( + '8688f7e11f63d8a7eac7cb87af850337fabbd400', + ); + + expect(hasteFS.getSha1(path.join(DIR, 'node_modules/bar/index.js'))).toBe( + 'ee245b9fbd45e1f6ad300eb2f5484844f6b5a34c', + ); + + // Ignored files do not get the SHA-1 computed. + + expect(hasteFS.getSha1(path.join(DIR, 'file_with_extension.ignored'))).toBe( + null, + ); + + expect( + hasteFS.getSha1( + path.join(DIR, 'node_modules/bar/file_with_extension.ignored'), + ), + ).toBe(null); +}); diff --git a/packages/jest-haste-map/src/__tests__/worker.test.js b/packages/jest-haste-map/src/__tests__/worker.test.js index 0d905fde16d2..13406384eb7b 100644 --- a/packages/jest-haste-map/src/__tests__/worker.test.js +++ b/packages/jest-haste-map/src/__tests__/worker.test.js @@ -14,7 +14,7 @@ import ConditionalTest from '../../../../scripts/ConditionalTest'; import H from '../constants'; -const {worker} = require('../worker'); +const {worker, getSha1} = require('../worker'); let mockFs; let readFileSync; @@ -47,10 +47,8 @@ describe('worker', () => { readFileSync = fs.readFileSync; fs.readFileSync = jest.fn((path, options) => { - expect(options).toBe('utf8'); - if (mockFs[path]) { - return mockFs[path]; + return options === 'utf8' ? mockFs[path] : Buffer.from(mockFs[path]); } throw new Error(`Cannot read path '${path}'.`); @@ -109,4 +107,22 @@ describe('worker', () => { expect(error.message).toEqual(`Cannot read path '/kiwi.js'.`); }); + + it('simply computes SHA-1s when requested', async () => { + expect( + await getSha1({computeSha1: false, filePath: '/fruits/banana.js'}), + ).toEqual({sha1: null}); + + expect( + await getSha1({computeSha1: true, filePath: '/fruits/banana.js'}), + ).toEqual({sha1: 'f24c6984cce6f032f6d55d771d04ab8dbbe63c8c'}); + + expect( + await getSha1({computeSha1: true, filePath: '/fruits/pear.js'}), + ).toEqual({sha1: '1bf6fc618461c19553e27f8b8021c62b13ff614a'}); + + await expect( + getSha1({computeSha1: true, filePath: '/i/dont/exist.js'}), + ).rejects.toThrow(); + }); }); diff --git a/packages/jest-haste-map/src/index.js b/packages/jest-haste-map/src/index.js index 8637caa1956a..161fd4970a10 100644 --- a/packages/jest-haste-map/src/index.js +++ b/packages/jest-haste-map/src/index.js @@ -9,7 +9,7 @@ import {execSync} from 'child_process'; import {version as VERSION} from '../package.json'; -import {worker} from './worker'; +import {getSha1, worker} from './worker'; import crypto from 'crypto'; import EventEmitter from 'events'; import getMockName from './get_mock_name'; @@ -86,7 +86,7 @@ type Watcher = { close(callback: () => void): void, }; -type WorkerInterface = {worker: typeof worker}; +type WorkerInterface = {worker: typeof worker, getSha1: typeof getSha1}; export type ModuleMap = HasteModuleMap; export type FS = HasteFS; @@ -407,9 +407,60 @@ class HasteMap extends EventEmitter { moduleMap[platform] = module; }; + const fileMetadata = hasteMap.files[filePath]; + const moduleMetadata = hasteMap.map[fileMetadata[H.ID]]; + const computeSha1 = this._options.computeSha1 && !fileMetadata[H.SHA1]; + + // Callback called when the response from the worker is successful. + const workerReply = metadata => { + // `1` for truthy values instead of `true` to save cache space. + fileMetadata[H.VISITED] = 1; + + const metadataId = metadata.id; + const metadataModule = metadata.module; + + if (metadataId && metadataModule) { + fileMetadata[H.ID] = metadataId; + setModule(metadataId, metadataModule); + } + + fileMetadata[H.DEPENDENCIES] = metadata.dependencies || []; + + if (computeSha1) { + fileMetadata[H.SHA1] = metadata.sha1; + } + }; + + // Callback called when the response from the worker is an error. + const workerError = error => { + if (typeof error !== 'object' || !error.message || !error.stack) { + error = new Error(error); + error.stack = ''; // Remove stack for stack-less errors. + } + + // $FlowFixMe: checking error code is OK if error comes from "fs". + if (!['ENOENT', 'EACCES'].includes(error.code)) { + throw error; + } + + // If a file cannot be read we remove it from the file list and + // ignore the failure silently. + delete hasteMap.files[filePath]; + }; + // If we retain all files in the virtual HasteFS representation, we avoid // reading them if they aren't important (node_modules). if (this._options.retainAllFiles && this._isNodeModulesDir(filePath)) { + if (computeSha1) { + return this._getWorker(workerOptions) + .getSha1({ + computeSha1, + filePath, + hasteImplModulePath: this._options.hasteImplModulePath, + }) + .then(workerReply, workerError); + } + return null; } @@ -433,10 +484,6 @@ class HasteMap extends EventEmitter { mocks[mockPath] = filePath; } - const fileMetadata = hasteMap.files[filePath]; - const moduleMetadata = hasteMap.map[fileMetadata[H.ID]]; - const computeSha1 = this._options.computeSha1 && !fileMetadata[H.SHA1]; - if (fileMetadata[H.VISITED]) { if (!fileMetadata[H.ID]) { return null; @@ -467,41 +514,7 @@ class HasteMap extends EventEmitter { filePath, hasteImplModulePath: this._options.hasteImplModulePath, }) - .then( - metadata => { - // `1` for truthy values instead of `true` to save cache space. - fileMetadata[H.VISITED] = 1; - - const metadataId = metadata.id; - const metadataModule = metadata.module; - - if (metadataId && metadataModule) { - fileMetadata[H.ID] = metadataId; - setModule(metadataId, metadataModule); - } - - fileMetadata[H.DEPENDENCIES] = metadata.dependencies || []; - - if (computeSha1) { - fileMetadata[H.SHA1] = metadata.sha1; - } - }, - error => { - if (typeof error !== 'object' || !error.message || !error.stack) { - error = new Error(error); - error.stack = ''; // Remove stack for stack-less errors. - } - - // $FlowFixMe: checking error code is OK if error comes from "fs". - if (['ENOENT', 'EACCES'].indexOf(error.code) < 0) { - throw error; - } - - // If a file cannot be read we remove it from the file list and - // ignore the failure silently. - delete hasteMap.files[filePath]; - }, - ); + .then(workerReply, workerError); } _buildHasteMap(data: { @@ -563,14 +576,14 @@ class HasteMap extends EventEmitter { _getWorker(options: ?{forceInBand: boolean}): WorkerInterface { if (!this._worker) { if ((options && options.forceInBand) || this._options.maxWorkers <= 1) { - this._worker = {worker}; + this._worker = {getSha1, worker}; } else { // $FlowFixMe: assignment of a worker with custom properties. this._worker = (new Worker(require.resolve('./worker'), { - exposedMethods: ['worker'], + exposedMethods: ['getSha1', 'worker'], maxRetries: 3, numWorkers: this._options.maxWorkers, - }): {worker: typeof worker}); + }): WorkerInterface); } } diff --git a/packages/jest-haste-map/src/worker.js b/packages/jest-haste-map/src/worker.js index 969e384dbbc2..372889ae0302 100644 --- a/packages/jest-haste-map/src/worker.js +++ b/packages/jest-haste-map/src/worker.js @@ -22,6 +22,13 @@ const PACKAGE_JSON = path.sep + 'package.json'; let hasteImpl: ?HasteImpl = null; let hasteImplModulePath: ?string = null; +function computeSha1(content) { + return crypto + .createHash('sha1') + .update(content) + .digest('hex'); +} + export async function worker(data: WorkerMessage): Promise { if ( data.hasteImplModulePath && @@ -76,11 +83,21 @@ export async function worker(data: WorkerMessage): Promise { content = fs.readFileSync(filePath); } - sha1 = crypto - .createHash('sha1') - .update(content) - .digest('hex'); + sha1 = computeSha1(content); } return {dependencies, id, module, sha1}; } + +export async function getSha1(data: WorkerMessage): Promise { + const sha1 = data.computeSha1 + ? computeSha1(fs.readFileSync(data.filePath)) + : null; + + return { + dependencies: undefined, + id: undefined, + module: undefined, + sha1, + }; +}