From 20d6b0aeecfb11e7857cdc96589d237ae69c0538 Mon Sep 17 00:00:00 2001 From: NullVoxPopuli Date: Wed, 10 Mar 2021 22:50:16 -0500 Subject: [PATCH] ref: #609 -- start esbuild prototype replacement for webpack --- packages/esbuild/jest.config.js | 6 + packages/esbuild/package.json | 38 ++ packages/esbuild/src/ember-webpack.ts | 551 +++++++++++++++++++++++ packages/esbuild/src/html-entrypoint.ts | 154 +++++++ packages/esbuild/src/html-placeholder.ts | 91 ++++ packages/esbuild/src/stat-summary.ts | 12 + yarn.lock | 46 +- 7 files changed, 894 insertions(+), 4 deletions(-) create mode 100644 packages/esbuild/jest.config.js create mode 100644 packages/esbuild/package.json create mode 100644 packages/esbuild/src/ember-webpack.ts create mode 100644 packages/esbuild/src/html-entrypoint.ts create mode 100644 packages/esbuild/src/html-placeholder.ts create mode 100644 packages/esbuild/src/stat-summary.ts diff --git a/packages/esbuild/jest.config.js b/packages/esbuild/jest.config.js new file mode 100644 index 0000000000..7f4f45dca5 --- /dev/null +++ b/packages/esbuild/jest.config.js @@ -0,0 +1,6 @@ +module.exports = { + testEnvironment: 'node', + testMatch: [ + '/tests/**/*.test.js', + ], +}; diff --git a/packages/esbuild/package.json b/packages/esbuild/package.json new file mode 100644 index 0000000000..ebf1ead94c --- /dev/null +++ b/packages/esbuild/package.json @@ -0,0 +1,38 @@ +{ + "name": "@embroider/esbuild", + "version": "0.37.0", + "private": false, + "description": "Builds EmberJS apps with ESBuild", + "repository": { + "type": "git", + "url": "https://github.com/embroider-build/embroider.git", + "directory": "packages/esbuild" + }, + "license": "MIT", + "author": "NullVoxPopuli", + "main": "src/ember-webpack.js", + "files": [ + "src/**/*.js", + "src/**/*.d.ts", + "src/**/*.js.map" + ], + "scripts": { + "prepare": "tsc" + }, + "dependencies": { + "esbuild": "^0.9.0" + }, + "devDependencies": { + "@types/node": "^10.5.2", + "typescript": "~4.0.0" + }, + "peerDependencies": { + "@embroider/core": "0.37.0" + }, + "engines": { + "node": "10.* || 12.* || >= 14" + }, + "volta": { + "extends": "../../package.json" + } +} diff --git a/packages/esbuild/src/ember-webpack.ts b/packages/esbuild/src/ember-webpack.ts new file mode 100644 index 0000000000..8eb0ef7c08 --- /dev/null +++ b/packages/esbuild/src/ember-webpack.ts @@ -0,0 +1,551 @@ +/* + Most of the work this module does is putting an HTML-oriented facade around + Webpack. That is, we want both the input and output to be primarily HTML files + with proper spec semantics, and we use webpack to optimize the assets referred + to by those files. + + While there are webpack plugins for handling HTML, none of them handle + multiple HTML entrypoints and apply correct HTML semantics (for example, + getting script vs module context correct). +*/ + +import { getOrCreate, Variant, applyVariantToBabelConfig } from '@embroider/core'; +import { PackagerInstance, AppMeta, Packager } from '@embroider/core'; +import webpack, { Configuration } from 'webpack'; +import { readFileSync, outputFileSync, copySync, realpathSync, Stats, statSync, readJsonSync } from 'fs-extra'; +import { join, dirname, relative, sep } from 'path'; +import isEqual from 'lodash/isEqual'; +import mergeWith from 'lodash/mergeWith'; +import flatMap from 'lodash/flatMap'; +import MiniCssExtractPlugin from 'mini-css-extract-plugin'; +import makeDebug from 'debug'; +import { format } from 'util'; +import { tmpdir } from 'os'; +import { warmup as threadLoaderWarmup } from 'thread-loader'; +import { HTMLEntrypoint } from './html-entrypoint'; +import { StatSummary } from './stat-summary'; +import crypto from 'crypto'; +import type { HbsLoaderConfig } from '@embroider/hbs-loader'; + +const debug = makeDebug('embroider:debug'); + +// This is a type-only import, so it gets compiled away. At runtime, we load +// terser lazily so it's only loaded for production builds that use it. Don't +// add any non-type-only imports here. +import type { MinifyOptions } from 'terser'; + +interface AppInfo { + entrypoints: HTMLEntrypoint[]; + otherAssets: string[]; + templateCompiler: AppMeta['template-compiler']; + babel: AppMeta['babel']; + rootURL: AppMeta['root-url']; + publicAssetURL: string; + resolvableExtensions: AppMeta['resolvable-extensions']; +} + +// AppInfos are equal if they result in the same webpack config. +function equalAppInfo(left: AppInfo, right: AppInfo): boolean { + return ( + isEqual(left.babel, right.babel) && + left.entrypoints.length === right.entrypoints.length && + left.entrypoints.every((e, index) => isEqual(e.modules, right.entrypoints[index].modules)) + ); +} + +interface Options { + webpackConfig: Configuration; + + // the base public URL for your assets in production. Use this when you want + // to serve all your assets from a different origin (like a CDN) than your + // actual index.html will be served on. + // + // This should be a URL ending in "/". + publicAssetURL?: string; +} + +// we want to ensure that not only does our instance conform to +// PackagerInstance, but our constructor conforms to Packager. So instead of +// just exporting our class directly, we export a const constructor of the +// correct type. +const Webpack: Packager = class Webpack implements PackagerInstance { + static annotation = '@embroider/webpack'; + + pathToVanillaApp: string; + private extraConfig: Configuration | undefined; + private passthroughCache: Map = new Map(); + private publicAssetURL: string | undefined; + + constructor( + pathToVanillaApp: string, + private outputPath: string, + private variants: Variant[], + private consoleWrite: (msg: string) => void, + options?: Options + ) { + this.pathToVanillaApp = realpathSync(pathToVanillaApp); + this.extraConfig = options?.webpackConfig; + this.publicAssetURL = options?.publicAssetURL; + warmUp(); + } + + async build(): Promise { + let appInfo = this.examineApp(); + let webpack = this.getWebpack(appInfo); + let stats = this.summarizeStats(await this.runWebpack(webpack)); + await this.writeFiles(stats, appInfo); + } + + private examineApp(): AppInfo { + let meta = JSON.parse(readFileSync(join(this.pathToVanillaApp, 'package.json'), 'utf8'))['ember-addon'] as AppMeta; + let templateCompiler = meta['template-compiler']; + let rootURL = meta['root-url']; + let babel = meta['babel']; + let resolvableExtensions = meta['resolvable-extensions']; + let entrypoints = []; + let otherAssets = []; + let publicAssetURL = this.publicAssetURL || rootURL; + + for (let relativePath of meta.assets) { + if (/\.html/i.test(relativePath)) { + entrypoints.push(new HTMLEntrypoint(this.pathToVanillaApp, rootURL, publicAssetURL, relativePath)); + } else { + otherAssets.push(relativePath); + } + } + + return { entrypoints, otherAssets, templateCompiler, babel, rootURL, resolvableExtensions, publicAssetURL }; + } + + private configureWebpack( + { entrypoints, templateCompiler, babel, resolvableExtensions, publicAssetURL }: AppInfo, + variant: Variant + ): Configuration { + let entry: { [name: string]: string } = {}; + for (let entrypoint of entrypoints) { + for (let moduleName of entrypoint.modules) { + entry[moduleName] = './' + moduleName; + } + } + + let hbsOptions: HbsLoaderConfig = { + templateCompilerFile: join(this.pathToVanillaApp, templateCompiler.filename), + variant, + }; + + return { + mode: variant.optimizeForProduction ? 'production' : 'development', + context: this.pathToVanillaApp, + entry, + performance: { + hints: false, + }, + plugins: [ + new MiniCssExtractPlugin({ + filename: `chunk.[chunkhash].css`, + chunkFilename: `chunk.[chunkhash].css`, + }), + ], + node: false, + module: { + rules: [ + { + test: /\.hbs$/, + use: nonNullArray([ + maybeThreadLoader(templateCompiler.isParallelSafe), + { + loader: require.resolve('@embroider/hbs-loader'), + options: hbsOptions, + }, + ]), + }, + { + // eslint-disable-next-line @typescript-eslint/no-require-imports + test: require(join(this.pathToVanillaApp, babel.fileFilter)), + use: nonNullArray([ + maybeThreadLoader(babel.isParallelSafe), + babelLoaderOptions(babel.majorVersion, variant, join(this.pathToVanillaApp, babel.filename)), + ]), + }, + { + test: isCSS, + use: this.makeCSSRule(variant), + }, + ], + }, + output: { + path: join(this.outputPath, 'assets'), + filename: `chunk.[chunkhash].js`, + chunkFilename: `chunk.[chunkhash].js`, + publicPath: publicAssetURL + 'assets/', + }, + optimization: { + splitChunks: { + chunks: 'all', + }, + }, + resolve: { + extensions: resolvableExtensions, + }, + resolveLoader: { + alias: { + // these loaders are our dependencies, not the app's dependencies. I'm + // not overriding the default loader resolution rules in case the app also + // wants to control those. + 'thread-loader': require.resolve('thread-loader'), + 'babel-loader-8': require.resolve('babel-loader'), + 'babel-loader-7': require.resolve('@embroider/babel-loader-7'), + 'css-loader': require.resolve('css-loader'), + 'style-loader': require.resolve('style-loader'), + }, + }, + }; + } + + private lastAppInfo: AppInfo | undefined; + private lastWebpack: webpack.MultiCompiler | undefined; + + private getWebpack(appInfo: AppInfo) { + if (this.lastWebpack && this.lastAppInfo && equalAppInfo(appInfo, this.lastAppInfo)) { + debug(`reusing webpack config`); + return this.lastWebpack; + } + debug(`configuring webpack`); + let config = this.variants.map(variant => + mergeWith({}, this.configureWebpack(appInfo, variant), this.extraConfig, appendArrays) + ); + this.lastAppInfo = appInfo; + return (this.lastWebpack = webpack(config)); + } + + private async writeScript(script: string, written: Set, variant: Variant) { + if (!variant.optimizeForProduction) { + this.copyThrough(script); + return script; + } + + // loading these lazily here so they never load in non-production builds. + // The node cache will ensures we only load them once. + const [Terser, srcURL] = await Promise.all([import('terser'), import('source-map-url')]); + + let inCode = readFileSync(join(this.pathToVanillaApp, script), 'utf8'); + let terserOpts: MinifyOptions = {}; + let fileRelativeSourceMapURL; + let appRelativeSourceMapURL; + if (srcURL.default.existsIn(inCode)) { + fileRelativeSourceMapURL = srcURL.default.getFrom(inCode)!; + appRelativeSourceMapURL = join(dirname(script), fileRelativeSourceMapURL); + let content; + try { + content = readJsonSync(join(this.pathToVanillaApp, appRelativeSourceMapURL)); + } catch (err) { + // the script refers to a sourcemap that doesn't exist, so we just leave + // the map out. + } + if (content) { + terserOpts.sourceMap = { content, url: fileRelativeSourceMapURL }; + } + } + let { code: outCode, map: outMap } = await Terser.default.minify(inCode, terserOpts); + let finalFilename = this.getFingerprintedFilename(script, outCode!); + outputFileSync(join(this.outputPath, finalFilename), outCode!); + written.add(script); + if (appRelativeSourceMapURL && outMap) { + outputFileSync(join(this.outputPath, appRelativeSourceMapURL), outMap); + written.add(appRelativeSourceMapURL); + } + return finalFilename; + } + + private async writeStyle(style: string, written: Set, variant: Variant) { + if (!variant.optimizeForProduction) { + this.copyThrough(style); + written.add(style); + return style; + } + + const csso = await import('csso'); + const cssContent = readFileSync(join(this.pathToVanillaApp, style), 'utf8'); + const minifiedCss = csso.minify(cssContent).css; + + let finalFilename = this.getFingerprintedFilename(style, minifiedCss); + outputFileSync(join(this.outputPath, finalFilename), minifiedCss); + written.add(style); + return finalFilename; + } + + private async provideErrorContext(message: string, messageParams: any[], fn: () => Promise) { + try { + return await fn(); + } catch (err) { + let context = format(message, ...messageParams); + err.message = context + ': ' + err.message; + throw err; + } + } + + private async writeFiles(stats: StatSummary, { entrypoints, otherAssets }: AppInfo) { + // we're doing this ourselves because I haven't seen a webpack 4 HTML plugin + // that handles multiple HTML entrypoints correctly. + + let written: Set = new Set(); + // scripts (as opposed to modules) and stylesheets (as opposed to CSS + // modules that are imported from JS modules) get passed through without + // going through webpack. + for (let entrypoint of entrypoints) { + await this.provideErrorContext('needed by %s', [entrypoint.filename], async () => { + for (let script of entrypoint.scripts) { + if (!stats.entrypoints.has(script)) { + try { + // zero here means we always attribute passthrough scripts to the + // first build variant + stats.entrypoints.set( + script, + new Map([[0, [await this.writeScript(script, written, this.variants[0])]]]) + ); + } catch (err) { + if (err.code === 'ENOENT' && err.path === join(this.pathToVanillaApp, script)) { + this.consoleWrite( + `warning: in ${entrypoint.filename}