diff --git a/packages/next/export/index.js b/packages/next/export/index.js index ef5c0ea263f7f..42bbb532b99e5 100644 --- a/packages/next/export/index.js +++ b/packages/next/export/index.js @@ -1,10 +1,15 @@ import { cpus } from 'os' -import { fork } from 'child_process' -import { recursiveCopy } from '../lib/recursive-copy' +import chalk from 'chalk' +import Worker from 'jest-worker' +import { promisify } from 'util' import mkdirpModule from 'mkdirp' import { resolve, join } from 'path' +import { API_ROUTE } from '../lib/constants' import { existsSync, readFileSync } from 'fs' -import chalk from 'chalk' +import createProgress from 'tty-aware-progress' +import { recursiveCopy } from '../lib/recursive-copy' +import { recursiveDelete } from '../lib/recursive-delete' +import { formatAmpMessages } from '../build/output/index' import loadConfig, { isTargetLikeServerless } from '../next-server/server/config' @@ -17,11 +22,6 @@ import { CLIENT_PUBLIC_FILES_PATH, CLIENT_STATIC_FILES_PATH } from '../next-server/lib/constants' -import createProgress from 'tty-aware-progress' -import { promisify } from 'util' -import { recursiveDelete } from '../lib/recursive-delete' -import { API_ROUTE } from '../lib/constants' -import { formatAmpMessages } from '../build/output/index' const mkdirp = promisify(mkdirpModule) @@ -33,7 +33,6 @@ export default async function (dir, options, configuration) { dir = resolve(dir) const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir) - const concurrency = options.concurrency || 10 const threads = options.threads || Math.max(cpus().length - 1, 1) const distDir = join(dir, nextConfig.distDir) const subFolders = nextConfig.exportTrailingSlash @@ -130,9 +129,7 @@ export default async function (dir, options, configuration) { nextExport: true } - log( - ` launching ${threads} threads with concurrency of ${concurrency} per thread` - ) + log(` launching ${threads} workers`) const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, { dev: false, dir, @@ -163,20 +160,6 @@ export default async function (dir, options, configuration) { const progress = !options.silent && createProgress(filteredPaths.length) - const chunks = filteredPaths.reduce((result, route, i) => { - const worker = i % threads - if (!result[worker]) { - result[worker] = { paths: [], pathMap: {} } - } - result[worker].pathMap[route] = exportPathMap[route] - result[worker].paths.push(route) - - if (options.sprPages && options.sprPages.has(route)) { - result[worker].pathMap[route].sprPage = true - } - return result - }, []) - const ampValidations = {} let hadValidationError = false @@ -195,46 +178,45 @@ export default async function (dir, options, configuration) { } }) } - const workers = new Set() + + const worker = new Worker(require.resolve('./worker'), { + maxRetries: 0, + numWorkers: threads, + enableWorkerThreads: true, + exposedMethods: ['default'] + }) + + worker.getStdout().pipe(process.stdout) + worker.getStderr().pipe(process.stderr) + + let renderError = false await Promise.all( - chunks.map( - chunk => - new Promise((resolve, reject) => { - const worker = fork(require.resolve('./worker'), [], { - env: process.env - }) - workers.add(worker) - worker.send({ - distDir, - buildId, - exportPaths: chunk.paths, - exportPathMap: chunk.pathMap, - outDir, - renderOpts, - serverRuntimeConfig, - concurrency, - subFolders, - serverless: isTargetLikeServerless(nextConfig.target) - }) - worker.on('message', ({ type, payload }) => { - if (type === 'progress' && progress) { - progress() - } else if (type === 'error') { - reject(payload) - } else if (type === 'done') { - resolve() - } else if (type === 'amp-validation') { - ampValidations[payload.page] = payload.result - hadValidationError = - hadValidationError || payload.result.errors.length - } - }) - }) - ) + filteredPaths.map(async path => { + const result = await worker.default({ + path, + pathMap: exportPathMap[path], + distDir, + buildId, + outDir, + renderOpts, + serverRuntimeConfig, + subFolders, + serverless: isTargetLikeServerless(nextConfig.target) + }) + + for (const validation of result.ampValidations || []) { + const { page, result } = validation + ampValidations[page] = result + hadValidationError |= + Array.isArray(result && result.errors) && result.errors.length > 0 + } + renderError |= result.error + if (progress) progress() + }) ) - workers.forEach(worker => worker.kill()) + worker.end() if (Object.keys(ampValidations).length) { console.log(formatAmpMessages(ampValidations)) @@ -245,6 +227,9 @@ export default async function (dir, options, configuration) { ) } + if (renderError) { + throw new Error(`Export encountered errors`) + } // Add an empty line to the console for the better readability. log('') } diff --git a/packages/next/export/worker.js b/packages/next/export/worker.js index 9252e0c81b2cd..ab553e9ab4ff0 100644 --- a/packages/next/export/worker.js +++ b/packages/next/export/worker.js @@ -3,7 +3,6 @@ import { promisify } from 'util' import { extname, join, dirname, sep } from 'path' import { renderToHTML } from '../next-server/server/render' import { writeFile, access } from 'fs' -import { Sema } from 'async-sema' import AmpHtmlValidator from 'amphtml-validator' import { loadComponents } from '../next-server/server/load-components' import { isDynamicRoute } from '../next-server/lib/router/utils/is-dynamic' @@ -11,192 +10,180 @@ import { getRouteMatcher } from '../next-server/lib/router/utils/route-matcher' import { getRouteRegex } from '../next-server/lib/router/utils/route-regex' const envConfig = require('../next-server/lib/runtime-config') -const mkdirp = promisify(mkdirpModule) const writeFileP = promisify(writeFile) +const mkdirp = promisify(mkdirpModule) const accessP = promisify(access) global.__NEXT_DATA__ = { nextExport: true } -process.on( - 'message', - async ({ - distDir, - buildId, - exportPaths, - exportPathMap, - outDir, - renderOpts, - serverRuntimeConfig, - concurrency, - subFolders, - serverless - }) => { - const sema = new Sema(concurrency, { capacity: exportPaths.length }) - try { - const work = async path => { - await sema.acquire() - let { query = {} } = exportPathMap[path] - const { page, sprPage } = exportPathMap[path] - const filePath = path === '/' ? '/index' : path - const ampPath = `${filePath}.amp` - - // Check if the page is a specified dynamic route - if (isDynamicRoute(page) && page !== path) { - const params = getRouteMatcher(getRouteRegex(page))(path) - if (params) { - query = { - ...query, - ...params - } - } else { - throw new Error( - `The provided export path '${path}' doesn't match the '${page}' page.\nRead more: https://err.sh/zeit/next.js/export-path-mismatch` - ) - } - } +export default async function ({ + path, + pathMap, + distDir, + buildId, + outDir, + renderOpts, + serverRuntimeConfig, + subFolders, + serverless +}) { + let results = { + ampValidations: [] + } - const headerMocks = { - headers: {}, - getHeader: () => ({}), - setHeader: () => {}, - hasHeader: () => false, - removeHeader: () => {}, - getHeaderNames: () => [] + try { + let { query = {} } = pathMap + const { page, sprPage } = pathMap + const filePath = path === '/' ? '/index' : path + const ampPath = `${filePath}.amp` + + // Check if the page is a specified dynamic route + if (isDynamicRoute(page) && page !== path) { + const params = getRouteMatcher(getRouteRegex(page))(path) + if (params) { + query = { + ...query, + ...params } + } else { + throw new Error( + `The provided export path '${path}' doesn't match the '${page}' page.\nRead more: https://err.sh/zeit/next.js/export-path-mismatch` + ) + } + } - const req = { - url: path, - ...headerMocks - } - const res = { - ...headerMocks - } + const headerMocks = { + headers: {}, + getHeader: () => ({}), + setHeader: () => {}, + hasHeader: () => false, + removeHeader: () => {}, + getHeaderNames: () => [] + } - if (sprPage && isDynamicRoute(page)) { - query._nextPreviewSkeleton = 1 - // pass via `req` to avoid adding code to serverless bundle - req.url += - (req.url.includes('?') ? '&' : '?') + '_nextPreviewSkeleton=1' - } + const req = { + url: path, + ...headerMocks + } + const res = { + ...headerMocks + } - envConfig.setConfig({ - serverRuntimeConfig, - publicRuntimeConfig: renderOpts.runtimeConfig - }) + if (sprPage && isDynamicRoute(page)) { + query._nextPreviewSkeleton = 1 + // pass via `req` to avoid adding code to serverless bundle + req.url += (req.url.includes('?') ? '&' : '?') + '_nextPreviewSkeleton=1' + } - let htmlFilename = `${filePath}${sep}index.html` - if (!subFolders) htmlFilename = `${filePath}.html` - - const pageExt = extname(page) - const pathExt = extname(path) - // Make sure page isn't a folder with a dot in the name e.g. `v1.2` - if (pageExt !== pathExt && pathExt !== '') { - // If the path has an extension, use that as the filename instead - htmlFilename = path - } else if (path === '/') { - // If the path is the root, just use index.html - htmlFilename = 'index.html' - } + envConfig.setConfig({ + serverRuntimeConfig, + publicRuntimeConfig: renderOpts.runtimeConfig + }) + + let htmlFilename = `${filePath}${sep}index.html` + if (!subFolders) htmlFilename = `${filePath}.html` + + const pageExt = extname(page) + const pathExt = extname(path) + // Make sure page isn't a folder with a dot in the name e.g. `v1.2` + if (pageExt !== pathExt && pathExt !== '') { + // If the path has an extension, use that as the filename instead + htmlFilename = path + } else if (path === '/') { + // If the path is the root, just use index.html + htmlFilename = 'index.html' + } - const baseDir = join(outDir, dirname(htmlFilename)) - const htmlFilepath = join(outDir, htmlFilename) + const baseDir = join(outDir, dirname(htmlFilename)) + const htmlFilepath = join(outDir, htmlFilename) + + await mkdirp(baseDir) + let html + let curRenderOpts = {} + let renderMethod = renderToHTML + + if (serverless) { + renderMethod = require(join( + distDir, + 'serverless/pages', + (page === '/' ? 'index' : page) + '.js' + )).renderReqToHTML + const result = await renderMethod(req, res, true) + curRenderOpts = result.renderOpts + html = result.html + } else { + const components = await loadComponents( + distDir, + buildId, + page, + serverless + ) + + if (typeof components.Component === 'string') { + html = components.Component + } else { + curRenderOpts = { ...components, ...renderOpts, ampPath } + html = await renderMethod(req, res, page, query, curRenderOpts) + } + } - await mkdirp(baseDir) - let html - let curRenderOpts = {} - let renderMethod = renderToHTML + const validateAmp = async (html, page) => { + const validator = await AmpHtmlValidator.getInstance() + const result = validator.validateString(html) + const errors = result.errors.filter(e => e.severity === 'ERROR') + const warnings = result.errors.filter(e => e.severity !== 'ERROR') + + if (warnings.length || errors.length) { + results.ampValidations.push({ + page, + result: { + errors, + warnings + } + }) + } + } + if (curRenderOpts.inAmpMode) { + await validateAmp(html, path) + } else if (curRenderOpts.hybridAmp) { + // we need to render the AMP version + let ampHtmlFilename = `${ampPath}${sep}index.html` + if (!subFolders) { + ampHtmlFilename = `${ampPath}.html` + } + const ampBaseDir = join(outDir, dirname(ampHtmlFilename)) + const ampHtmlFilepath = join(outDir, ampHtmlFilename) + + try { + await accessP(ampHtmlFilepath) + } catch (_) { + // make sure it doesn't exist from manual mapping + let ampHtml if (serverless) { - renderMethod = require(join( - distDir, - 'serverless/pages', - (page === '/' ? 'index' : page) + '.js' - )).renderReqToHTML - const result = await renderMethod(req, res, true) - curRenderOpts = result.renderOpts - html = result.html + req.url += (req.url.includes('?') ? '&' : '?') + 'amp=1' + ampHtml = (await renderMethod(req, res, true)).html } else { - const components = await loadComponents( - distDir, - buildId, + ampHtml = await renderMethod( + req, + res, page, - serverless + { ...query, amp: 1 }, + curRenderOpts ) - - if (typeof components.Component === 'string') { - html = components.Component - } else { - curRenderOpts = { ...components, ...renderOpts, ampPath } - html = await renderMethod(req, res, page, query, curRenderOpts) - } } - const validateAmp = async (html, page) => { - const validator = await AmpHtmlValidator.getInstance() - const result = validator.validateString(html) - const errors = result.errors.filter(e => e.severity === 'ERROR') - const warnings = result.errors.filter(e => e.severity !== 'ERROR') - - if (warnings.length || errors.length) { - process.send({ - type: 'amp-validation', - payload: { - page, - result: { - errors, - warnings - } - } - }) - } - } - - if (curRenderOpts.inAmpMode) { - await validateAmp(html, path) - } else if (curRenderOpts.hybridAmp) { - // we need to render the AMP version - let ampHtmlFilename = `${ampPath}${sep}index.html` - if (!subFolders) { - ampHtmlFilename = `${ampPath}.html` - } - const ampBaseDir = join(outDir, dirname(ampHtmlFilename)) - const ampHtmlFilepath = join(outDir, ampHtmlFilename) - - try { - await accessP(ampHtmlFilepath) - } catch (_) { - // make sure it doesn't exist from manual mapping - let ampHtml - if (serverless) { - req.url += (req.url.includes('?') ? '&' : '?') + 'amp=1' - ampHtml = (await renderMethod(req, res, true)).html - } else { - ampHtml = await renderMethod( - req, - res, - page, - { ...query, amp: 1 }, - curRenderOpts - ) - } - - await validateAmp(ampHtml, page + '?amp=1') - await mkdirp(ampBaseDir) - await writeFileP(ampHtmlFilepath, ampHtml, 'utf8') - } - } - - await writeFileP(htmlFilepath, html, 'utf8') - process.send({ type: 'progress' }) - sema.release() + await validateAmp(ampHtml, page + '?amp=1') + await mkdirp(ampBaseDir) + await writeFileP(ampHtmlFilepath, ampHtml, 'utf8') } - await Promise.all(exportPaths.map(work)) - process.send({ type: 'done' }) - } catch (err) { - console.error(err) - process.send({ type: 'error', payload: err }) } + await writeFileP(htmlFilepath, html, 'utf8') + return results + } catch (error) { + console.error(`\nError occurred prerendering ${path}:`, error) + return { ...results, error: true } } -) +} diff --git a/test/integration/handles-export-errors/pages/index copy 10.js b/test/integration/handles-export-errors/pages/index copy 10.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 10.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 11.js b/test/integration/handles-export-errors/pages/index copy 11.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 11.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 12.js b/test/integration/handles-export-errors/pages/index copy 12.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 12.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 13.js b/test/integration/handles-export-errors/pages/index copy 13.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 13.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 2.js b/test/integration/handles-export-errors/pages/index copy 2.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 2.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 3.js b/test/integration/handles-export-errors/pages/index copy 3.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 3.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 4.js b/test/integration/handles-export-errors/pages/index copy 4.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 4.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 5.js b/test/integration/handles-export-errors/pages/index copy 5.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 5.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 6.js b/test/integration/handles-export-errors/pages/index copy 6.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 6.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 7.js b/test/integration/handles-export-errors/pages/index copy 7.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 7.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 8.js b/test/integration/handles-export-errors/pages/index copy 8.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 8.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy 9.js b/test/integration/handles-export-errors/pages/index copy 9.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy 9.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index copy.js b/test/integration/handles-export-errors/pages/index copy.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index copy.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/pages/index.js b/test/integration/handles-export-errors/pages/index.js new file mode 100644 index 0000000000000..c485e84a4c6a5 --- /dev/null +++ b/test/integration/handles-export-errors/pages/index.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +export default () => hello.world diff --git a/test/integration/handles-export-errors/test/index.test.js b/test/integration/handles-export-errors/test/index.test.js new file mode 100644 index 0000000000000..d859ee24ddd22 --- /dev/null +++ b/test/integration/handles-export-errors/test/index.test.js @@ -0,0 +1,18 @@ +/* eslint-env jest */ +/* global jasmine */ +import path from 'path' +import { nextBuild } from 'next-test-utils' + +jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5 +const appDir = path.join(__dirname, '..') + +describe('Handles Errors During Export', () => { + it('Does not crash workers', async () => { + const { stdout, stderr } = await nextBuild(appDir, [], { + stdout: true, + stderr: true + }) + + expect(stdout + stderr).not.toMatch(/ERR_IPC_CHANNEL_CLOSED/) + }) +})