feat(ssr): remove external files dependencies (#10885)

Before the ssr dist had dependencies on external files (e.g. index.html).

By in-lining those dependencies we can generate a portable standalone main.js.

Author: fiji-flo

Diff for: .env.testing

@@ -34,3 +34,5 @@ BUILD_ALWAYS_ALLOW_ROBOTS=true
 REACT_APP_ENABLE_PLUS=true
 
 REACT_APP_FXA_SIGNIN_URL=/users/fxa/login/authenticate/
+
+BASE_URL="https://developer.mozilla.org"

Diff for: .gitignore

@@ -47,6 +47,7 @@ yarn-error.log*
 /server/*.js.map
 /ssr/dist/
 /ssr/*.js
+!/ssr/mozilla.dnthelper.min.js
 !/ssr/webpack.config.js
 /ssr/*.js.map
 /tool/*.js

Diff for: client/config/webpack.config.js

@@ -95,7 +95,7 @@ function config(webpackEnv) {
   // as %PUBLIC_URL% in `index.html` and `process.env.PUBLIC_URL` in JavaScript.
   // Omit trailing slash as %PUBLIC_URL%/xyz looks better than %PUBLIC_URL%xyz.
   // Get environment variables to inject into our app.
-  const env = getClientEnvironment(paths.publicUrlOrPath.slice(0, -1));
+  const env = getClientEnvironment(paths.publicUrlOrPath.replace(/\/$/, ""));
 
   const shouldUseReactRefresh = env.raw.FAST_REFRESH;
 

Diff for: client/public/index.html

@@ -9,13 +9,13 @@
       of the file that has a hash in it.
     -->
 
-    <link rel="icon" href="%PUBLIC_URL%/favicon-48x48.png" />
+    <link rel="icon" href="/favicon-48x48.png" />
 
-    <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png" />
+    <link rel="apple-touch-icon" href="/apple-touch-icon.png" />
 
     <meta name="theme-color" content="#ffffff" />
 
-    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
+    <link rel="manifest" href="/manifest.json" />
 
     <link
       rel="search"
@@ -58,7 +58,7 @@
       property="og:description"
       content="The MDN Web Docs site provides information about Open Web technologies including HTML, CSS, and APIs for both Web sites and progressive web apps."
     />
-    <meta property="og:image" content="%PUBLIC_URL%/mdn-social-share.png" />
+    <meta property="og:image" content="/mdn-social-share.png" />
     <meta property="og:image:type" content="image/png" />
     <meta property="og:image:height" content="1080" />
     <meta property="og:image:width" content="1920" />

Diff for: client/scripts/build.js

@@ -15,6 +15,7 @@ import printBuildError from "react-dev-utils/printBuildError.js";
 
 import configFactory from "../config/webpack.config.js";
 import paths from "../config/paths.js";
+import { hashSomeStaticFilesForClientBuild } from "./postprocess-client-build.js";
 
 // Makes the script crash on unhandled rejections instead of silently
 // ignoring them. In the future, promise rejections that are not handled will
@@ -120,6 +121,17 @@ checkBrowsers(paths.appPath, isInteractive)
       }
     }
   )
+  .then(async () => {
+    const { results } = await hashSomeStaticFilesForClientBuild(paths.appBuild);
+    console.log(
+      chalk.green(
+        `Hashed ${results.length} files in ${path.join(
+          paths.appBuild,
+          "index.html"
+        )}`
+      )
+    );
+  })
   .catch((err) => {
     if (err && err.message) {
       console.log(err.message);

Diff for: tool/optimize-client-build.ts renamed to client/scripts/postprocess-client-build.js

@@ -9,7 +9,7 @@ import path from "node:path";
 import cheerio from "cheerio";
 import md5File from "md5-file";
 
-export async function runOptimizeClientBuild(buildRoot) {
+export async function hashSomeStaticFilesForClientBuild(buildRoot) {
   const indexHtmlFilePath = path.join(buildRoot, "index.html");
   const indexHtml = fs.readFileSync(indexHtmlFilePath, "utf-8");
 
@@ -26,17 +26,11 @@ export async function runOptimizeClientBuild(buildRoot) {
       if (element.attribs.property !== "og:image") {
         return;
       }
+      // This is a can of worms. Using from environment for now.
+      // We need to use an absolute URL for "og:image".
+      hrefPrefix = process.env.BASE_URL || "";
       href = element.attribs.content;
       attributeKey = "content";
-      // This is an unfortunate hack. The value for the
-      // <meta property=og:image content=...> needs to be an absolute URL.
-      // We tested with a relative URL and it seems it doesn't work in Twitter.
-      // So we hardcode the URL to be our production domain so the URL is
-      // always absolute.
-      // Yes, this makes it a bit weird to use a build of this on a dev,
-      // stage, preview, or a local build. Especially if the hashed URL doesn't
-      // always work. But it's a fair price to pay.
-      hrefPrefix = "https://developer.mozilla.org";
     } else {
       href = element.attribs.href;
       if (!href) {
@@ -75,7 +69,7 @@ export async function runOptimizeClientBuild(buildRoot) {
     const splitName = filePath.split(extName);
     const hashedFilePath = `${splitName[0]}.${hash}${extName}`;
     fs.copyFileSync(filePath, hashedFilePath);
-    const hashedHref = filePathToHref(buildRoot, hashedFilePath);
+    const hashedHref = filePathToHref(buildRoot, hashedFilePath, href);
     results.push({
       filePath,
       href,
@@ -101,8 +95,19 @@ export async function runOptimizeClientBuild(buildRoot) {
 }
 
 // Turn 'C:\Path\to\client\build\favicon.ico' to '/favicon.ico'
-function filePathToHref(root, filePath) {
-  return `${filePath.replace(root, "").replace(path.sep, "/")}`;
+// or 'https://foo.bar/favicon.ico' if href is an absolute URL.
+function filePathToHref(root, filePath, href) {
+  let dummyOrExistingUrl = new URL(href, "http://localhost.example");
+  dummyOrExistingUrl.pathname = "";
+  let url = new URL(
+    `${filePath.replace(root, "").replace(path.sep, "/")}`,
+    dummyOrExistingUrl
+  );
+  if (url.hostname === "localhost.example") {
+    return url.pathname;
+  } else {
+    return url.href;
+  }
 }
 
 // Turn '/favicon.ico' to 'C:\Path\to\client\build\favicon.ico'

Diff for: client/scripts/start.js

@@ -32,7 +32,7 @@ process.on("unhandledRejection", (err) => {
 
 const appPackageJson = JSON.parse(fs.readFileSync(paths.appPackageJson));
 
-const env = getClientEnvironment(paths.publicUrlOrPath.slice(0, -1));
+const env = getClientEnvironment(paths.publicUrlOrPath.replace(/\/$/, ""));
 const useYarn = fs.existsSync(paths.yarnLockFile);
 const isInteractive = process.stdout.isTTY;
 

Diff for: client/src/homepage/contributor-spotlight/index.tsx

@@ -5,9 +5,7 @@ import { Icon } from "../../ui/atoms/icon";
 import Mandala from "../../ui/molecules/mandala";
 
 import "./index.scss";
-const contributorGraphic = `${
-  process.env.PUBLIC_URL || ""
-}/assets/mdn_contributor.png`;
+const contributorGraphic = "/assets/mdn_contributor.png";
 
 export function ContributorSpotlight(props: HydrationData<any>) {
   const fallbackData = props.hyData ? props : undefined;

Diff for: client/src/ui/atoms/avatar/index.tsx

@@ -6,7 +6,7 @@ export const Avatar = ({ userData }: { userData: UserData }) => {
   // If we have user data and the user is logged in, show their
   // profile pic, defaulting to the dino head if the avatar
   // URL doesn't work.
-  const avatarImage = `${process.env.PUBLIC_URL || ""}/assets/avatar.png`;
+  const avatarImage = "/assets/avatar.png";
 
   return (
     <div

Diff for: package.json

@@ -22,8 +22,8 @@
     "build:curriculum": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/build-curriculum.ts",
     "build:dist": "tsc -p tsconfig.dist.json",
     "build:glean": "cd client && cross-env VIRTUAL_ENV=venv glean translate src/telemetry/metrics.yaml src/telemetry/pings.yaml -f typescript -o src/telemetry/generated",
-    "build:prepare": "yarn build:client && yarn build:ssr && yarn tool optimize-client-build && yarn tool google-analytics-code && yarn tool popularities && yarn tool spas && yarn tool gather-git-history && yarn tool build-robots-txt",
-    "build:ssr": "cd ssr && webpack --mode=production",
+    "build:prepare": "yarn build:client && yarn build:ssr && yarn tool popularities && yarn tool spas && yarn tool gather-git-history && yarn tool build-robots-txt",
+    "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ssr/prepare.ts && cd ssr && webpack --mode=production",
     "build:sw": "cd client/pwa && yarn && yarn build:prod",
     "build:sw-dev": "cd client/pwa && yarn && yarn build",
     "check:tsc": "find . -name 'tsconfig.json' ! -wholename '**/node_modules/**' -print0 | xargs -n1 -P 2 -0 sh -c 'cd `dirname $0` && echo \"🔄 $(pwd)\" && npx tsc --noEmit && echo \"☑️  $(pwd)\" || exit 255'",

Diff for: ssr/.gitignore

@@ -0,0 +1 @@
+include.ts

Diff for: ssr/ga.ts

@@ -0,0 +1,47 @@
+import fs from "node:fs";
+import {
+  BUILD_OUT_ROOT,
+  GOOGLE_ANALYTICS_MEASUREMENT_ID,
+} from "../libs/env/index.js";
+import path from "node:path";
+
+export async function generateGA() {
+  const outFile = path.join(BUILD_OUT_ROOT, "static", "js", "gtag.js");
+  const measurementIds =
+    GOOGLE_ANALYTICS_MEASUREMENT_ID.split(",").filter(Boolean);
+  if (measurementIds.length) {
+    const dntHelperCode = fs
+      .readFileSync(
+        new URL("mozilla.dnthelper.min.js", import.meta.url),
+        "utf-8"
+      )
+      .trim();
+
+    const firstMeasurementId = measurementIds[0];
+    const gaScriptURL = `https://www.googletagmanager.com/gtag/js?id=${encodeURIComponent(firstMeasurementId)}`;
+
+    const code = `
+// Mozilla DNT Helper
+${dntHelperCode}
+// Load GA unless DNT is enabled.
+if (Mozilla && !Mozilla.dntEnabled()) {
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+  ${measurementIds
+    .map((id) => `gtag('config', '${id}', { 'anonymize_ip': true });`)
+    .join("\n  ")}
+
+  var gaScript = document.createElement('script');
+  gaScript.async = true;
+  gaScript.src = '${gaScriptURL}';
+  document.head.appendChild(gaScript);
+}`.trim();
+    fs.writeFileSync(outFile, `${code}\n`, "utf-8");
+    console.log(
+      `Generated ${outFile} for SSR rendering using ${GOOGLE_ANALYTICS_MEASUREMENT_ID}.`
+    );
+  } else {
+    console.log("No Google Analytics code file generated");
+  }
+}

Diff for: ssr/include.d.ts

@@ -0,0 +1,4 @@
+export const WEBFONT_TAGS: string;
+export const GTAG_PATH: null | string;
+export const BASE_URL: string;
+export const ALWAYS_ALLOW_ROBOTS: boolean;

Diff for: ssr/index.ts

@@ -1,19 +1,9 @@
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-import * as dotenv from "dotenv";
 import React from "react";
 import { StaticRouter } from "react-router-dom/server";
 
 import { App } from "../client/src/app";
 import render from "./render";
 
-const dirname = fileURLToPath(new URL(".", import.meta.url));
-
-dotenv.config({
-  path: path.join(dirname, "..", process.env.ENV_FILE || ".env"),
-});
-
 export function renderHTML(url, context) {
   return render(
     React.createElement(

Diff for: tool/mozilla.dnthelper.min.js renamed to ssr/mozilla.dnthelper.min.js

@@ -0,0 +1,73 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  ALWAYS_ALLOW_ROBOTS,
+  BUILD_OUT_ROOT,
+  BASE_URL,
+} from "../libs/env/index.js";
+import { generateGA } from "./ga.js";
+
+const dirname = path.dirname(fileURLToPath(new URL(".", import.meta.url)));
+const clientBuildRoot = path.resolve(dirname, "client/build");
+
+function extractWebFontURLs() {
+  const urls: string[] = [];
+  const manifest = JSON.parse(
+    fs.readFileSync(path.join(clientBuildRoot, "asset-manifest.json"), "utf-8")
+  );
+  for (const entrypoint of manifest.entrypoints) {
+    if (!entrypoint.endsWith(".css")) continue;
+    const css = fs.readFileSync(
+      path.join(clientBuildRoot, entrypoint),
+      "utf-8"
+    );
+    const generator = extractCSSURLs(css, (url) => url.endsWith(".woff2"));
+    urls.push(...generator);
+  }
+  return [...new Set(urls)];
+}
+
+function* extractCSSURLs(css, filterFunction) {
+  for (const match of css.matchAll(/url\((.*?)\)/g)) {
+    const url = match[1];
+    if (filterFunction(url)) {
+      yield url;
+    }
+  }
+}
+
+function webfontTags(webfontURLs): string {
+  return webfontURLs
+    .map(
+      (url) =>
+        `<link rel="preload" as="font" type="font/woff2" href="${url}" crossorigin>`
+    )
+    .join("");
+}
+
+function gtagScriptPath(relPath = "/static/js/gtag.js") {
+  const filePath = relPath.split("/").slice(1).join(path.sep);
+  if (fs.existsSync(path.join(BUILD_OUT_ROOT, filePath))) {
+    return relPath;
+  }
+  return null;
+}
+
+function prepare() {
+  const webfontURLs = extractWebFontURLs();
+  const tags = webfontTags(webfontURLs);
+  const gtagPath = gtagScriptPath();
+
+  fs.writeFileSync(
+    path.join(dirname, "ssr", "include.ts"),
+    `
+export const WEBFONT_TAGS = ${JSON.stringify(tags)};
+export const GTAG_PATH = ${JSON.stringify(gtagPath)};
+export const BASE_URL = ${JSON.stringify(BASE_URL)};
+export const ALWAYS_ALLOW_ROBOTS = ${JSON.stringify(ALWAYS_ALLOW_ROBOTS)};
+`
+  );
+}
+
+generateGA().then(() => prepare());