feat(build): split rendering index.html from building index.json (#10953)

DEPRECATION: Rendering index.html files as part of the build command 
is now deprecated, and will no longer be supported in Yari v3. 
To resolve the deprecation warning, add the `-n` (`--nohtml`) option.

Problem: Currently, building the `index.json` files and rendering the
`index.html` files happens in a single step via the `yarn build` command.
Although it's possible to only build the `index.json` files via `yarn build -n`,
it is not possible to only render the `index.html` files.

Solution:
1. Split rendering the `index.html` files into a separate command (`yarn render:html`).
2. Introduce a new explicit command for building the JSON files for docs (`yarn build:docs`).
3. Reuse the `render:html` command to render static pages (aka "SPAs"), by ensuring the corresponding builds produce `index.json`s (that just contain a `pageTitle` and a `url` for now).
4. Show a deprecation warning (see above) when `yarn build` is called **without** the `-n` flag.

Co-authored-by: Claas Augner <[email protected]>

Author: fiji-flo

Diff for: .github/workflows/dev-build.yml

@@ -141,9 +141,9 @@ jobs:
           # Spread the work across 2 processes. Why 2? Because that's what you
           # get in the default GitHub hosting Linux runners.
           # See https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-          yarn build --locale     en-us --locale     ja --locale     fr &
+          yarn build:docs --locale     en-us --locale     ja --locale     fr &
           build1=$!
-          yarn build --not-locale en-us --not-locale ja --not-locale fr &
+          yarn build:docs --not-locale en-us --not-locale ja --not-locale fr &
           build2=$!
 
           # You must explicitly specify the job you're waiting-on to ensure
@@ -159,6 +159,9 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
+          # SSR all pages
+          yarn render:html
+
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json

Diff for: .github/workflows/performance.yml

@@ -53,8 +53,9 @@ jobs:
           BUILD_GOOGLE_ANALYTICS_MEASUREMENT_ID: G-XXXXXXXX
         run: |
           yarn build:prepare
-          # BUILD_FOLDERSEARCH=mdn/kitchensink yarn build
-          BUILD_FOLDERSEARCH=web/javascript/reference/global_objects/array/foreach yarn build
+          # BUILD_FOLDERSEARCH=mdn/kitchensink yarn build:docs
+          BUILD_FOLDERSEARCH=web/javascript/reference/global_objects/array/foreach yarn build:docs
+          yarn render:html
 
       - name: Serve and lhci
         env:

Diff for: .github/workflows/prod-build.yml

@@ -264,7 +264,7 @@ jobs:
           # Build using one process per locale.
           # Note: We have 4 cores, but 9 processes is a reasonable number.
           for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do
-            yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
+            yarn build:docs --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
             pids+=($!)
           done
 
@@ -283,6 +283,9 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
+          # SSR all pages
+          yarn render:html
+
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json

Diff for: .github/workflows/stage-build.yml

@@ -280,7 +280,7 @@ jobs:
           # Build using one process per locale.
           # Note: We have 4 cores, but 9 processes is a reasonable number.
           for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do
-            yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
+            yarn build:docs --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
             pids+=($!)
           done
 
@@ -299,6 +299,9 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
+          # SSR all pages
+          yarn render:html
+
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json

Diff for: .github/workflows/testing.yml

@@ -90,7 +90,8 @@ jobs:
           ENV_FILE: .env.testing
         run: |
           yarn build:prepare
-          yarn build
+          yarn build:docs
+          yarn render:html
 
           yarn start:static-server > /tmp/stdout.log 2> /tmp/stderr.log &
           sleep 1

Diff for: .github/workflows/xyz-build.yml

@@ -171,7 +171,7 @@ jobs:
           # Build using one process per locale.
           # Note: We have 4 cores, but 9 processes is a reasonable number.
           for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do
-            yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
+            yarn build:docs --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
             pids+=($!)
           done
 
@@ -190,6 +190,9 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
+          # SSR all pages
+          yarn render:html
+
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json

Diff for: build/blog.ts

@@ -18,9 +18,6 @@ import {
   AuthorFrontmatter,
   AuthorMetadata,
 } from "../libs/types/blog.js";
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore
-import { renderHTML } from "../ssr/dist/main.js";
 import {
   findPostFileBySlug,
   injectLoadingLazyAttributes,
@@ -240,24 +237,24 @@ export async function buildBlogIndex(options: { verbose?: boolean }) {
   const prefix = "blog";
   const locale = DEFAULT_LOCALE;
 
-  const hyData = {
-    posts: await allPostFrontmatter(),
+  const context: HydrationData = {
+    hyData: {
+      posts: await allPostFrontmatter(),
+    },
+    pageTitle: "MDN Blog",
+    url: `/${locale}/${prefix}/`,
   };
-  const context = { hyData, pageTitle: "MDN Blog" };
 
-  const html = renderHTML(`/${locale}/${prefix}/`, context);
   const outPath = path.join(BUILD_OUT_ROOT, locale.toLowerCase(), `${prefix}`);
 
   await fs.mkdir(outPath, { recursive: true });
-  const filePath = path.join(outPath, "index.html");
   const jsonFilePath = path.join(outPath, "index.json");
 
   await fs.mkdir(outPath, { recursive: true });
-  await fs.writeFile(filePath, html);
   await fs.writeFile(jsonFilePath, JSON.stringify(context));
 
   if (options.verbose) {
-    console.log("Wrote", filePath);
+    console.log("Wrote", jsonFilePath);
   }
 }
 
@@ -280,8 +277,8 @@ export async function buildBlogPosts(options: {
       continue;
     }
 
-    const url = `/${locale}/blog/${blogMeta.slug}/`;
-    const renderUrl = `/${locale}/blog/${blogMeta.slug}`;
+    const url = `/${locale}/${prefix}/${blogMeta.slug}/`;
+    const renderUrl = `/${locale}/${prefix}/${blogMeta.slug}`;
     const renderDoc: BlogPostDoc = {
       url: renderUrl,
       rawBody: body,
@@ -303,6 +300,7 @@ export async function buildBlogPosts(options: {
       locale,
       noIndexing: options.noIndexing,
       image: blogMeta.image?.file && `${BASE_URL}${url}${blogMeta.image?.file}`,
+      url,
     };
 
     const outPath = path.join(
@@ -330,17 +328,13 @@ export async function buildBlogPosts(options: {
       await fs.copyFile(from, to);
     }
 
-    const html = renderHTML(`/${locale}/${prefix}/${blogMeta.slug}/`, context);
-
-    const filePath = path.join(outPath, "index.html");
     const jsonFilePath = path.join(outPath, "index.json");
 
     await fs.mkdir(outPath, { recursive: true });
-    await fs.writeFile(filePath, html);
     await fs.writeFile(jsonFilePath, JSON.stringify(context));
 
     if (options.verbose) {
-      console.log("Wrote", filePath);
+      console.log("Wrote", jsonFilePath);
     }
   }
 }

Diff for: build/cli.ts

@@ -18,7 +18,6 @@ import {
 import { DEFAULT_LOCALE, VALID_LOCALES } from "../libs/constants/index.js";
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
-import { renderHTML } from "../ssr/dist/main.js";
 import options from "./build-options.js";
 import {
   buildDocument,
@@ -32,6 +31,8 @@ import { humanFileSize } from "./utils.js";
 import { initSentry } from "./sentry.js";
 import { macroRenderTimes } from "../kumascript/src/render.js";
 import { fdir } from "fdir";
+import { ssrDocument } from "./ssr.js";
+import { HydrationData } from "../libs/types/hydration.js";
 
 const { program } = caporal;
 const { prompt } = inquirer;
@@ -240,20 +241,21 @@ async function buildDocuments(
       updateBaselineBuildMetadata(builtDocument);
     }
 
+    const context: HydrationData = {
+      doc: builtDocument,
+      url: builtDocument.mdn_url,
+    };
+
     if (!noHTML) {
-      fs.writeFileSync(
-        path.join(outPath, "index.html"),
-        renderHTML(document.url, { doc: builtDocument })
-      );
+      fs.writeFileSync(path.join(outPath, "index.html"), ssrDocument(context));
     }
-
     if (plainHTML) {
       fs.writeFileSync(path.join(outPath, "plain.html"), plainHTML);
     }
 
     // This is exploiting the fact that renderHTML has the side-effect of
     // mutating the built document which makes this not great and refactor-worthy.
-    const docString = JSON.stringify({ doc: builtDocument });
+    const docString = JSON.stringify(context);
     fs.writeFileSync(path.join(outPath, "index.json"), docString);
     fs.writeFileSync(
       path.join(outPath, "contributors.txt"),
@@ -472,7 +474,7 @@ program
   .option("-i, --interactive", "Ask what to do when encountering flaws", {
     default: false,
   })
-  .option("-n, --nohtml", "Do not build index.html", {
+  .option("-n, --nohtml", "Do not render index.html", {
     default: false,
   })
   .option("-l, --locale <locale...>", "Filtered specific locales", {
@@ -489,6 +491,12 @@ program
   .argument("[files...]", "specific files to build")
   .action(async ({ args, options }: BuildArgsAndOptions) => {
     try {
+      if (!options.nohtml) {
+        console.warn(
+          "WARNING: Rendering index.html files as part of the build command is now DEPRECATED, and will no longer be supported in Yari v3. To resolve this warning, add the `-n` (`--nohtml`) option. For details, see: https://github.com/mdn/yari/pull/10953"
+        );
+      }
+
       if (!options.quiet) {
         const roots = [
           ["CONTENT_ROOT", CONTENT_ROOT],

Diff for: build/curriculum.ts

@@ -37,7 +37,6 @@ import { HydrationData } from "../libs/types/hydration.js";
 import { memoize, slugToFolder } from "../content/utils.js";
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
-import { renderHTML } from "../ssr/dist/main.js";
 import { CheerioAPI } from "cheerio";
 import { buildSitemap } from "./sitemaps.js";
 
@@ -368,6 +367,7 @@ export async function buildCurriculum(options: {
       pageTitle: meta.title,
       locale,
       noIndexing: options.noIndexing,
+      url: `/${locale}/${meta.slug}/`,
     };
 
     const outPath = path.join(
@@ -378,17 +378,13 @@ export async function buildCurriculum(options: {
 
     await fs.mkdir(outPath, { recursive: true });
 
-    const html: string = renderHTML(`/${locale}/${meta.slug}/`, context);
-
-    const filePath = path.join(outPath, "index.html");
     const jsonFilePath = path.join(outPath, "index.json");
 
     await fs.mkdir(outPath, { recursive: true });
-    await fs.writeFile(filePath, html);
     await fs.writeFile(jsonFilePath, JSON.stringify(context));
 
     if (options.verbose) {
-      console.log("Wrote", filePath);
+      console.log("Wrote", jsonFilePath);
     }
   }
 }