Combine .css.proxy.js files into one (#1407)

* Add CSS preloading to @snowpack/plugin-optimize

* Add test snapshot

* Fix JS transform issue

* Add CSS Module support

* Replace accidental test deletion

* Update plugin-optimize snapshot

* Add preloadedCSSName option to plugin, docs

* Fix Node 10

* Fix link tag, multi-line HTML comment bug

* Fix CSS min bug

* Clean up

* Clean up already-imported CSS files

* Remove unused exclude option

* Update config name

* Fix optimize test

* Skip one test on Windows

plugins/plugin-optimize/README.md

@@ -29,10 +29,12 @@ Then add this plugin to your Snowpack config:
 
 ### Plugin Options
 
-| Name             |       Type        | Description                                                                                                                                                                                                                                                    |
-| :--------------- | :---------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `minifyJS`       |     `boolean`     | Enable JS minification (default: `true`)                                                                                                                                                                                                                       |
-| `minifyCSS`      |     `boolean`     | Enable CSS minification (default: `true`)                                                                                                                                                                                                                      |
-| `minifyHTML`     |     `boolean`     | Enable HTML minification (default: `true`)                                                                                                                                                                                                                     |
-| `preloadModules` |     `boolean`     | Experimental: Add deep, optimized [`<link rel="modulepreload">`](https://developers.google.com/web/updates/2017/12/modulepreload) tags into your HTML. (default: `false`)                                                                                      |
-| `target`         | `string,string[]` | The language target(s) to transpile to. This can be a single string (ex: "es2018") or an array of strings (ex: ["chrome58","firefox57"]). If undefined, no transpilation will be done. See [esbuild documentation](https://github.com/evanw/esbuild) for more. |
+| Name                 |       Type        | Description                                                                                                                                                                                                                                                    |
+| :------------------- | :---------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `minifyJS`           |     `boolean`     | Enable JS minification (default: `true`)                                                                                                                                                                                                                       |
+| `minifyCSS`          |     `boolean`     | Enable CSS minification (default: `true`)                                                                                                                                                                                                                      |
+| `minifyHTML`         |     `boolean`     | Enable HTML minification (default: `true`)                                                                                                                                                                                                                     |
+| `preloadModules`     |     `boolean`     | Experimental: Add deep, optimized [`<link rel="modulepreload">`](https://developers.google.com/web/updates/2017/12/modulepreload) tags into your HTML. (default: `false`)                                                                                      |
+| `preloadCSS`         |     `boolean`     | Experimental: Eliminate `.css.proxy.js` files and combine imported CSS into one file for better network performance (default: `false`)                                                                                                                         |
+| `preloadCSSFileName` |     `string`      | If preloading CSS, change the name of the generated CSS file. Only used in conjunction with `preloadCSS: true` (default: `/imported-styles.css`)                                                                                                               |
+| `target`             | `string,string[]` | The language target(s) to transpile to. This can be a single string (ex: "es2018") or an array of strings (ex: ["chrome58","firefox57"]). If undefined, no transpilation will be done. See [esbuild documentation](https://github.com/evanw/esbuild) for more. |

plugins/plugin-optimize/lib/css.js

@@ -0,0 +1,122 @@
+const fs = require('fs');
+const path = require('path');
+const {parse} = require('es-module-lexer');
+const csso = require('csso');
+
+/** Early-exit function that determines, given a set of JS files, if CSS is being imported */
+function hasCSSImport(files) {
+  for (const file of files) {
+    const code = fs.readFileSync(file, 'utf-8');
+    const [imports] = parse(code);
+    for (const {s, e} of imports.filter(({d}) => d === -1)) {
+      const spec = code.substring(s, e);
+      if (spec.endsWith('.css.proxy.js')) return true; // exit as soon as we find one
+    }
+  }
+  return false;
+}
+exports.hasCSSImport = hasCSSImport;
+
+/**
+ * Scans JS for CSS imports, and embeds only what’s needed
+ *
+ * import 'global.css'                       -> (removed; loaded in HTML)
+ * import url from 'global.css'              -> const url = 'global.css'
+ * import {foo, bar} from 'local.module.css' -> const {foo, bar} = 'local.module.css'
+ */
+function transformCSSProxy(file, originalCode) {
+  const filePath = path.dirname(file);
+  let code = originalCode;
+
+  const getProxyImports = (code) =>
+    parse(code)[0]
+      .filter(({d}) => d === -1) // discard dynamic imports (> -1) and import.meta (-2)
+      .filter(({s, e}) => code.substring(s, e).endsWith('.css.proxy.js')); // only accept .css.proxy.js files
+
+  // iterate through proxy imports
+  let proxyImports = getProxyImports(code);
+  while (proxyImports.length) {
+    const {s, e, ss, se} = proxyImports[0]; // only transform one at a time, because every transformation requires re-parsing (unless you created an ellaborate mechanism to keep track of character counts but IMO parsing is simpler/cheaper)
+
+    const originalImport = code.substring(s, e);
+    const importedFile = originalImport.replace(/\.proxy\.js$/, '');
+    const importNamed = code
+      .substring(ss, se)
+      .replace(code.substring(s - 1, e + 1), '') // remove import
+      .replace(/^import\s+/, '') // remove keyword
+      .replace(/\s*from.*$/, '') // remove other keyword
+      .replace(/\*\s+as\s+/, '') // sanitize star imports
+      .trim();
+
+    // transform JS
+    if (!importNamed) {
+      // option 1: no transforms needed
+      code = code.replace(new RegExp(`${code.substring(ss, se)};?\n?`), '');
+    } else {
+      if (importedFile.endsWith('.module.css')) {
+        // option 2: transform css modules
+        const proxyCode = fs.readFileSync(path.resolve(filePath, originalImport), 'utf-8');
+        const matches = proxyCode.match(/^let json\s*=\s*(\{[^\}]+\})/m);
+        if (matches) {
+          code = code.replace(
+            new RegExp(`${code.substring(ss, se).replace(/\*/g, '\\*')};?`),
+            `const ${importNamed.replace(/\*\s+as\s+/, '')} = ${matches[1]};`,
+          );
+        }
+      } else {
+        // option 3: transfrom normal css
+        code = code.replace(
+          new RegExp(`${code.substring(ss, se)};?`),
+          `const ${importNamed} = '${importedFile}';`,
+        );
+      }
+    }
+
+    proxyImports = getProxyImports(code); // re-parse code, continuing until all are transformed
+  }
+
+  return code;
+}
+exports.transformCSSProxy = transformCSSProxy;
+
+/** Build CSS File */
+function buildImportCSS(manifest, minifyCSS) {
+  // gather list of imported CSS files
+  const allCSSFiles = new Set();
+  for (const f in manifest) {
+    manifest[f].js.forEach((js) => {
+      if (!js.endsWith('.css.proxy.js')) return;
+      const isCSSModule = js.endsWith('.module.css.proxy.js');
+      allCSSFiles.add(isCSSModule ? js : js.replace(/\.proxy\.js$/, ''));
+    });
+  }
+
+  // read + concat
+  let code = '';
+  allCSSFiles.forEach((file) => {
+    const contents = fs.readFileSync(file, 'utf-8');
+
+    if (file.endsWith('.module.css.proxy.js')) {
+      // css modules
+      const matches = contents.match(/^export let code = *(.*)$/m);
+      if (matches && matches[1])
+        code +=
+          '\n' +
+          matches[1]
+            .trim()
+            .replace(/^('|")/, '')
+            .replace(/('|");?$/, '');
+    } else {
+      // normal css
+      code += '\n' + contents;
+      fs.unlinkSync(file); // after we‘ve scanned a CSS file, remove it (so it‘s not double-loaded)
+    }
+  });
+
+  // sanitize JSON values
+  const css = code.replace(/\\n/g, '\n').replace(/\\"/g, '"');
+
+  // minify
+  return minifyCSS ? csso.minify(css).css : css;
+}
+exports.buildImportCSS = buildImportCSS;

plugins/plugin-optimize/lib/html.js

@@ -0,0 +1,123 @@
+/**
+ * Logic for optimizing .html files (note: this will )
+ */
+const fs = require('fs');
+const path = require('path');
+const hypertag = require('hypertag');
+const {injectHTML} = require('node-inject-html');
+const {projectURL, isRemoteModule} = require('../util');
+const {scanJS} = require('./js');
+
+/** Scan HTML for static imports */
+async function scanHTML(htmlFiles, buildDirectory) {
+  const importList = {};
+  await Promise.all(
+    htmlFiles.map(async (htmlFile) => {
+      // TODO: add debug in plugins?
+      // log(`scanning ${projectURL(file, buildDirectory)} for imports`, 'debug');
+
+      const allCSSImports = new Set(); // all CSS imports for this HTML file
+      const allJSImports = new Set(); // all JS imports for this HTML file
+      const entry = new Set(); // keep track of HTML entry files
+
+      const code = await fs.promises.readFile(htmlFile, 'utf-8');
+
+      // <link>
+      hypertag(code, 'link').forEach((link) => {
+        if (!link.href) return;
+        if (isRemoteModule(link.href)) {
+          allCSSImports.add(link.href);
+        } else {
+          const resolvedCSS =
+            link.href[0] === '/'
+              ? path.join(buildDirectory, link.href)
+              : path.join(path.dirname(htmlFile), link.href);
+          allCSSImports.add(resolvedCSS);
+        }
+      });
+
+      // <script>
+      hypertag(code, 'script').forEach((script) => {
+        if (!script.src) return;
+        if (isRemoteModule(script.src)) {
+          allJSImports.add(script.src);
+        } else {
+          const resolvedJS =
+            script.src[0] === '/'
+              ? path.join(buildDirectory, script.src)
+              : path.join(path.dirname(htmlFile), script.src);
+          allJSImports.add(resolvedJS);
+          entry.add(resolvedJS);
+        }
+      });
+
+      // traverse all JS for other static imports (scannedFiles keeps track of files so we never redo work)
+      const scannedFiles = new Set();
+      allJSImports.forEach((jsFile) => {
+        scanJS({
+          file: jsFile,
+          rootDir: buildDirectory,
+          scannedFiles,
+          importList: allJSImports,
+        }).forEach((i) => allJSImports.add(i));
+      });
+
+      // return
+      importList[htmlFile] = {
+        entry: Array.from(entry),
+        css: Array.from(allCSSImports),
+        js: Array.from(allJSImports),
+      };
+    }),
+  );
+  return importList;
+}
+exports.scanHTML = scanHTML;
+
+/** Given a set of HTML files, trace the imported JS */
+function preloadJS({code, file, preloadCSS, rootDir}) {
+  const originalEntries = new Set(); // original entry files in HTML
+  const allModules = new Set(); // all modules required by this HTML file
+
+  // 1. scan HTML for <script> tags
+  hypertag(code, 'script').forEach((script) => {
+    if (!script.type || script.type !== 'module' || !script.src) return;
+    const resolvedJS =
+      script.src[0] === '/'
+        ? path.join(rootDir, script.src)
+        : path.join(path.dirname(file), script.src);
+    originalEntries.add(resolvedJS);
+  });
+
+  // 2. scan entries for additional imports
+  const scannedFiles = new Set(); // keep track of files scanned so we don’t get stuck in a circular dependency
+  originalEntries.forEach((entry) => {
+    scanJS({
+      file: entry,
+      rootDir,
+      scannedFiles,
+      importList: allModules,
+    }).forEach((file) => allModules.add(file));
+  });
+
+  // 3. add module preload to HTML (https://developers.google.com/web/updates/2017/12/modulepreload)
+  const resolvedModules = [...allModules]
+    .filter((m) => !originalEntries.has(m)) // don’t double-up preloading scripts that were already in the HTML
+    .filter((m) => (preloadCSS ? !m.endsWith('.css.proxy.js') : true)) // if preloading CSS, don’t preload .css.proxy.js
+    .map((src) => projectURL(src, rootDir));
+  if (!resolvedModules.length) return code; // don’t add useless whitespace
+  resolvedModules.sort((a, b) => a.localeCompare(b));
+
+  // 4. return HTML with preloads added
+  return injectHTML(code, {
+    headEnd:
+      `<!-- [@snowpack/plugin-optimize] Add modulepreload to improve unbundled load performance (More info: https://developers.google.com/web/updates/2017/12/modulepreload) -->\n` +
+      resolvedModules.map((src) => `    <link rel="modulepreload" href="${src}" />`).join('\n') +
+      '\n',
+    bodyEnd:
+      `<!-- [@snowpack/plugin-optimize] modulepreload fallback for browsers that do not support it yet -->\n    ` +
+      resolvedModules.map((src) => `<script type="module" src="${src}"></script>`).join('') +
+      '\n',
+  });
+}
+exports.preloadJS = preloadJS;

plugins/plugin-optimize/lib/js.js

@@ -0,0 +1,49 @@
+/**
+ * Functions for dealing with parsing/transforming JS
+ */
+const fs = require('fs');
+const path = require('path');
+const {parse} = require('es-module-lexer');
+const colors = require('kleur/colors');
+const {log, projectURL, isRemoteModule} = require('../util');
+
+/** Recursively scan JS for static imports */
+function scanJS({file, rootDir, scannedFiles, importList}) {
+  try {
+    // 1. scan file for static imports
+    scannedFiles.add(file); // keep track of scanned files so we never redo work
+    importList.add(file); // make sure import is marked
+    let code = fs.readFileSync(file, 'utf-8');
+    const [imports] = parse(code);
+    imports
+      .filter(({d}) => d === -1) // this is where we discard dynamic imports (> -1) and import.meta (-2)
+      .forEach(({s, e}) => {
+        const specifier = code.substring(s, e);
+        if (isRemoteModule(specifier)) {
+          importList.add(specifier);
+          scannedFiles.add(specifier); // don’t scan remote modules
+        } else {
+          importList.add(
+            specifier.startsWith('/')
+              ? path.join(rootDir, file)
+              : path.resolve(path.dirname(file), specifier),
+          );
+        }
+      });
+
+    // 2. recursively scan imports not yet scanned
+    [...importList]
+      .filter((fileLoc) => !scannedFiles.has(fileLoc)) // prevent infinite loop
+      .forEach((fileLoc) => {
+        scanJS({file: fileLoc, rootDir, scannedFiles, importList}).forEach((newImport) => {
+          importList.add(newImport);
+        });
+      });
+
+    return importList;
+  } catch (err) {
+    log(colors.yellow(` could not locate "${projectURL(file, rootDir)}"`), 'warn');
+    return importList;
+  }
+}
+exports.scanJS = scanJS;

plugins/plugin-optimize/package.json

@@ -13,12 +13,15 @@
     "access": "public"
   },
   "dependencies": {
-    "csso": "^4.0.3",
-    "es-module-lexer": "^0.3.24",
+    "csso": "^4.1.0",
+    "es-module-lexer": "^0.3.25",
     "esbuild": "^0.8.0",
     "glob": "^7.1.6",
     "html-minifier": "^4.0.0",
-    "kleur": "^4.1.1",
+    "hypertag": "^0.0.3",
+    "kleur": "^4.1.3",
+    "node-inject-html": "^0.0.5",
+    "object.fromentries": "^2.0.2",
     "p-queue": "^6.6.1"
   }
 }