From f81b6d56e03d3f883e8c5faebae44208bd1a328b Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 24 Aug 2019 13:55:43 -0500 Subject: [PATCH] Replace worker-farm with jest-worker (#8496) * Replace worker-farm with jest-worker * Apply suggestions from code review Co-Authored-By: Joe Haddad * Remove semaphores on top of jest-worker, unwind terser worker, and remove extra error log --- packages/next/build/index.ts | 32 ++------ packages/next/build/static-checker.ts | 17 ++--- .../terser-webpack-plugin/src/TaskRunner.js | 74 ++++++------------- .../terser-webpack-plugin/src/worker.js | 21 ------ packages/next/package.json | 5 +- yarn.lock | 12 ++- 6 files changed, 50 insertions(+), 111 deletions(-) delete mode 100644 packages/next/build/webpack/plugins/terser-webpack-plugin/src/worker.js diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index 358867a6e0519..708074bc5d8e6 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -1,4 +1,3 @@ -import { Sema } from 'async-sema' import chalk from 'chalk' import fs from 'fs' import mkdirpOrig from 'mkdirp' @@ -15,7 +14,7 @@ import loadConfig, { import nanoid from 'next/dist/compiled/nanoid/index.js' import path from 'path' import { promisify } from 'util' -import workerFarm from 'worker-farm' +import Worker from 'jest-worker' import formatWebpackMessages from '../client/dev/error-overlay/format-webpack-messages' import { recursiveDelete } from '../lib/recursive-delete' @@ -204,16 +203,10 @@ export default async function build(dir: string, conf = null): Promise { process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD - const staticCheckSema = new Sema(config.experimental.cpus, { - capacity: pageKeys.length, + const staticCheckWorkers = new Worker(staticCheckWorker, { + numWorkers: config.experimental.cpus, + enableWorkerThreads: true, }) - const staticCheckWorkers = workerFarm( - { - maxConcurrentWorkers: config.experimental.cpus, - }, - staticCheckWorker, - ['default'] - ) await Promise.all( pageKeys.map(async page => { @@ -268,17 +261,10 @@ export default async function build(dir: string, conf = null): Promise { if (nonReservedPage) { try { - await staticCheckSema.acquire() - const result: any = await new Promise((resolve, reject) => { - staticCheckWorkers.default( - { serverBundle, runtimeEnvConfig }, - (error: Error | null, result: any) => { - if (error) return reject(error) - resolve(result || {}) - } - ) + let result: any = await (staticCheckWorkers as any).default({ + serverBundle, + runtimeEnvConfig, }) - staticCheckSema.release() if (result.isHybridAmp) { hybridAmpPages.add(page) @@ -293,15 +279,13 @@ export default async function build(dir: string, conf = null): Promise { } catch (err) { if (err.message !== 'INVALID_DEFAULT_EXPORT') throw err invalidPages.add(page) - staticCheckSema.release() } } pageInfos.set(page, { size, chunks, serverBundle, static: isStatic }) }) ) - - workerFarm.end(staticCheckWorkers) + staticCheckWorkers.end() if (invalidPages.size > 0) { throw new Error( diff --git a/packages/next/build/static-checker.ts b/packages/next/build/static-checker.ts index bfdfe9911c9d8..371f55082d1df 100644 --- a/packages/next/build/static-checker.ts +++ b/packages/next/build/static-checker.ts @@ -1,14 +1,9 @@ import { isPageStatic } from './utils' -export default function worker( - options: any, - callback: (err: Error | null, data?: any) => void -) { - try { - const { serverBundle, runtimeEnvConfig } = options || ({} as any) - const result = isPageStatic(serverBundle, runtimeEnvConfig) - callback(null, result) - } catch (error) { - callback(error) - } +export default function worker(options: { + serverBundle: string + runtimeEnvConfig: any +}) { + const { serverBundle, runtimeEnvConfig } = options || ({} as any) + return isPageStatic(serverBundle, runtimeEnvConfig) } diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js index 4febfc2a4bef5..b306966292ede 100644 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js +++ b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js @@ -1,13 +1,11 @@ import { join } from 'path' import minify from './minify' import { promisify } from 'util' -import workerFarm from 'worker-farm' +import Worker from 'jest-worker' import { writeFile, readFile } from 'fs' -import serialize from 'serialize-javascript' -import { Sema } from 'async-sema' import mkdirp from 'mkdirp' -const worker = require.resolve('./worker') +const worker = require.resolve('./minify') const writeFileP = promisify(writeFile) const readFileP = promisify(readFile) @@ -16,11 +14,9 @@ export default class TaskRunner { if (cache) { mkdirp.sync((this.cacheDir = join(distDir, 'cache', 'next-minifier'))) } - // In some cases cpus() returns undefined // https://github.com/nodejs/node/issues/19022 this.maxConcurrentWorkers = cpus - this.sema = new Sema(cpus * 3) } run(tasks, callback) { @@ -31,36 +27,18 @@ export default class TaskRunner { } if (this.maxConcurrentWorkers > 1) { - const workerOptions = - process.platform === 'win32' - ? { - maxConcurrentWorkers: this.maxConcurrentWorkers, - maxConcurrentCallsPerWorker: 1, - } - : { maxConcurrentWorkers: this.maxConcurrentWorkers } - this.workers = workerFarm(workerOptions, worker) - this.boundWorkers = (options, cb) => { - try { - this.workers(serialize(options), cb) - } catch (error) { - // worker-farm can fail with ENOMEM or something else - cb(error) - } - } + this.workers = new Worker(worker, { + enableWorkerThreads: true, + numWorkers: this.maxConcurrentWorkers, + }) + this.boundWorkers = options => this.workers.default(options) } else { - this.boundWorkers = (options, cb) => { - try { - cb(null, minify(options)) - } catch (error) { - cb(error) - } - } + this.boundWorkers = async options => minify(options) } let toRun = tasks.length const results = [] const step = (index, data) => { - this.sema.release() toRun -= 1 results[index] = data @@ -71,37 +49,33 @@ export default class TaskRunner { tasks.forEach((task, index) => { const cachePath = this.cacheDir && join(this.cacheDir, task.cacheKey) - - const enqueue = () => { - this.boundWorkers(task, (error, data) => { - const result = error ? { error } : data + const enqueue = async () => { + try { + const result = await this.boundWorkers(task) const done = () => step(index, result) - - if (this.cacheDir && !result.error) { - writeFileP(cachePath, JSON.stringify(data), 'utf8') + if (cachePath) { + writeFileP(cachePath, JSON.stringify(result), 'utf8') .then(done) .catch(done) - } else { - done() } - }) + } catch (error) { + step(index, { error }) + } } - this.sema.acquire().then(() => { - if (this.cacheDir) { - readFileP(cachePath, 'utf8') - .then(data => step(index, JSON.parse(data))) - .catch(() => enqueue()) - } else { - enqueue() - } - }) + if (this.cacheDir) { + readFileP(cachePath, 'utf8') + .then(data => step(index, JSON.parse(data))) + .catch(() => enqueue()) + } else { + enqueue() + } }) } exit() { if (this.workers) { - workerFarm.end(this.workers) + this.workers.end() } } } diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/worker.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/worker.js deleted file mode 100644 index 8f0e6af9a304d..0000000000000 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/worker.js +++ /dev/null @@ -1,21 +0,0 @@ -import minify from './minify' - -module.exports = (options, callback) => { - try { - // 'use strict' => this === undefined (Clean Scope) - // Safer for possible security issues, albeit not critical at all here - // eslint-disable-next-line no-new-func, no-param-reassign - options = new Function( - 'exports', - 'require', - 'module', - '__filename', - '__dirname', - `'use strict'\nreturn ${options}` - )(exports, require, module, __filename, __dirname) - - callback(null, minify(options)) - } catch (errors) { - callback(errors) - } -} diff --git a/packages/next/package.json b/packages/next/package.json index 7b38bb1c2218c..8c7ed8e10adf7 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -80,6 +80,7 @@ "find-up": "4.0.0", "fork-ts-checker-webpack-plugin": "1.3.4", "fresh": "0.5.2", + "jest-worker": "24.9.0", "launch-editor": "2.2.1", "loader-utils": "1.2.3", "mkdirp": "0.5.1", @@ -88,7 +89,6 @@ "prop-types-exact": "1.2.0", "react-error-overlay": "5.1.6", "react-is": "16.8.6", - "serialize-javascript": "1.7.0", "source-map": "0.6.1", "string-hash": "1.1.3", "strip-ansi": "5.2.0", @@ -101,8 +101,7 @@ "webpack": "4.39.0", "webpack-dev-middleware": "3.7.0", "webpack-hot-middleware": "2.25.0", - "webpack-sources": "1.3.0", - "worker-farm": "1.7.0" + "webpack-sources": "1.3.0" }, "peerDependencies": { "react": "^16.6.0", diff --git a/yarn.lock b/yarn.lock index e55ac3e39002c..9715bfbdddf3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8345,6 +8345,14 @@ jest-watcher@^24.8.0: jest-util "^24.8.0" string-length "^2.0.0" +jest-worker@24.9.0: + version "24.9.0" + resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.9.0.tgz#5dbfdb5b2d322e98567898238a9697bcce67b3e5" + integrity sha512-51PE4haMSXcHohnSMdM42anbvZANYTqMrr52tVKPqqsPJMzoP6FYYDVqahX/HrAoKEKz3uUPzSvKs9A3qR4iVw== + dependencies: + merge-stream "^2.0.0" + supports-color "^6.1.0" + jest-worker@^24.6.0: version "24.6.0" resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.6.0.tgz#7f81ceae34b7cde0c9827a6980c35b7cdc0161b3" @@ -12315,7 +12323,7 @@ sentence-case@^2.1.0: no-case "^2.2.0" upper-case-first "^1.1.2" -serialize-javascript@1.7.0, serialize-javascript@^1.7.0: +serialize-javascript@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-1.7.0.tgz#d6e0dfb2a3832a8c94468e6eb1db97e55a192a65" integrity sha512-ke8UG8ulpFOxO8f8gRYabHQe/ZntKlcig2Mp+8+URDP1D8vJZ0KUt7LYo07q25Z/+JVSgpr/cui9PIp5H6/+nA== @@ -14267,7 +14275,7 @@ wordwrap@~1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= -worker-farm@1.7.0, worker-farm@^1.7.0: +worker-farm@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8" integrity sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==