perf: normalize engines static query context (#37764)

* perf: normalize engines static query context

Co-authored-by: LekoArts <[email protected]>

* drop unused

* add some filtering

* pass only slice static queries by template

* e2e case

* fix passing slice template -> sq mapping

* drop debug log

* add e2e case for ssg

---------

Co-authored-by: LekoArts <[email protected]>

Author: pieh

e2e-tests/production-runtime/cypress/integration/slices/slices.js

@@ -41,4 +41,10 @@ describe(`Slices`, () => {
       .invoke(`text`)
       .should(`contain`, `2`)
   })
+
+  it(`Slice with static query works`, () => {
+    cy.getTestElement(`footer-static-query-title`).contains(
+      `Gatsby Default Starter`
+    )
+  })
 })

e2e-tests/production-runtime/cypress/integration/ssr.js

@@ -3,8 +3,11 @@ const paramPath = `/ssr/param-path/`
 const wildcardPath = `/ssr/wildcard-path/`
 const pathRaking = `/ssr/path-ranking/`
 
-Cypress.on('uncaught:exception', (err, runnable) => {
-  if (err.message.includes('Minified React error #418') || err.message.includes('Minified React error #423')) {
+Cypress.on("uncaught:exception", (err, runnable) => {
+  if (
+    err.message.includes("Minified React error #418") ||
+    err.message.includes("Minified React error #423")
+  ) {
     return false
   }
 })
@@ -50,6 +53,13 @@ describe(`Static path ('${staticPath}')`, () => {
       expect(win.location.search).to.equal(queryString)
     })
   })
+
+  it(`Slice with static query works`, () => {
+    cy.visit(staticPath).waitForRouteChange()
+    cy.getTestElement(`footer-static-query-title`).contains(
+      `Gatsby Default Starter`
+    )
+  })
 })
 
 describe(`Param path ('${paramPath}:param')`, () => {

e2e-tests/production-runtime/src/components/footer.js

@@ -1,21 +1,42 @@
 import React, { useContext } from "react"
+import { useStaticQuery, graphql } from "gatsby"
 import { AppContext } from "../app-context"
 
 // Use as a Slice
-function Footer({ framework, lang, sliceContext: { framework: frameworkViaContext }}) {
+function Footer({
+  framework,
+  lang,
+  sliceContext: { framework: frameworkViaContext },
+}) {
   const { posts } = useContext(AppContext)
 
+  const data = useStaticQuery(graphql`
+    {
+      site {
+        siteMetadata {
+          alias: title
+        }
+      }
+    }
+  `)
+
   return (
     <footer
       style={{
         marginTop: `10px`,
         fontSize: `12px`,
       }}
     >
-     <span data-testid="footer-slice-context-value">{frameworkViaContext}</span>
-     <span data-testid="footer-static-text">Built with {` `}</span>
-     <span data-testid="footer-props">{`${framework}${lang}`}</span>
-     {` `}Posts Count: <span data-testid="footer-context-derieved-value">{`${posts.length}`}</span>
+      <span data-testid="footer-slice-context-value">
+        {frameworkViaContext}
+      </span>
+      <span data-testid="footer-static-text">Built with {` `}</span>
+      <span data-testid="footer-props">{`${framework}${lang}`}</span>
+      {` `}Posts Count:{" "}
+      <span data-testid="footer-context-derieved-value">{`${posts.length}`}</span>
+      <span data-testid="footer-static-query-title">
+        {data.site.siteMetadata.alias}
+      </span>
     </footer>
   )
 }

e2e-tests/production-runtime/src/pages/ssr/static-path.js

@@ -1,4 +1,5 @@
 import React from "react"
+import { Slice } from "gatsby"
 
 const UseEnv = ({ heading, envVar }) => (
   <React.Fragment>
@@ -30,7 +31,7 @@ export default function StaticPath({ serverData }) {
         heading="process.env.FROM_COMMAND_LINE"
         envVar={process.env.FROM_COMMAND_LINE}
       />
-       <UseEnv
+      <UseEnv
         heading="process.env.GATSBY_PREFIXED_FROM_COMMAND_LINE"
         envVar={process.env.GATSBY_PREFIXED_FROM_COMMAND_LINE}
       />
@@ -51,10 +52,11 @@ export default function StaticPath({ serverData }) {
         heading="serverData.envVars.FROM_COMMAND_LINE"
         envVar={serverData?.envVars?.FROM_COMMAND_LINE}
       />
-       <UseEnv
+      <UseEnv
         heading="serverData.envVars.GATSBY_PREFIXED_FROM_COMMAND_LINE"
         envVar={serverData?.envVars?.GATSBY_PREFIXED_FROM_COMMAND_LINE}
       />
+      <Slice alias="footer" framework="Gatsby" lang="js" />
     </div>
   )
 }
@@ -63,7 +65,8 @@ export async function getServerData(arg) {
   const VERY_SECRET_ALIAS_VAR = process.env.VERY_SECRET_VAR
   const EXISTING_VAR = process.env.EXISTING_VAR
   const FROM_COMMAND_LINE = process.env.FROM_COMMAND_LINE
-  const GATSBY_PREFIXED_FROM_COMMAND_LINE = process.env.GATSBY_PREFIXED_FROM_COMMAND_LINE
+  const GATSBY_PREFIXED_FROM_COMMAND_LINE =
+    process.env.GATSBY_PREFIXED_FROM_COMMAND_LINE
 
   return {
     props: {
@@ -72,7 +75,7 @@ export async function getServerData(arg) {
         VERY_SECRET_ALIAS_VAR,
         EXISTING_VAR,
         FROM_COMMAND_LINE,
-        GATSBY_PREFIXED_FROM_COMMAND_LINE
+        GATSBY_PREFIXED_FROM_COMMAND_LINE,
       },
     },
   }

packages/gatsby/src/commands/build-html.ts

@@ -778,11 +778,18 @@ export async function buildSlices({
     try {
       const slices = Array.from(state.slices.entries())
 
+      const staticQueriesBySliceTemplate = {}
+      for (const slice of state.slices.values()) {
+        staticQueriesBySliceTemplate[slice.componentPath] =
+          state.staticQueriesByTemplate.get(slice.componentPath)
+      }
+
       await workerPool.single.renderSlices({
         publicDir: path.join(program.directory, `public`),
         htmlComponentRendererPath,
         slices,
         slicesProps,
+        staticQueriesBySliceTemplate,
       })
     } catch (err) {
       const prettyError = createErrorFromString(

packages/gatsby/src/commands/build.ts

@@ -56,7 +56,7 @@ import {
 import { createGraphqlEngineBundle } from "../schema/graphql-engine/bundle-webpack"
 import {
   createPageSSRBundle,
-  writeQueryContext,
+  copyStaticQueriesToEngine,
 } from "../utils/page-ssr-module/bundle-webpack"
 import { shouldGenerateEngines } from "../utils/engines-helpers"
 import reporter from "gatsby-cli/lib/reporter"
@@ -379,15 +379,38 @@ module.exports = async function build(
     })
   }
 
-  // create scope so we don't leak state object
+  const engineTemplatePaths = new Set<string>()
   {
-    const state = store.getState()
-    await writeQueryContext({
-      staticQueriesByTemplate: state.staticQueriesByTemplate,
-      components: state.components,
+    let SSGCount = 0
+    let DSGCount = 0
+    let SSRCount = 0
+
+    for (const page of store.getState().pages.values()) {
+      if (page.mode === `SSR`) {
+        SSRCount++
+        engineTemplatePaths.add(page.componentPath)
+      } else if (page.mode === `DSG`) {
+        DSGCount++
+        engineTemplatePaths.add(page.componentPath)
+      } else {
+        SSGCount++
+      }
+    }
+
+    telemetry.addSiteMeasurement(`BUILD_END`, {
+      totalPagesCount: store.getState().pages.size, // total number of pages
+      SSRCount,
+      DSGCount,
+      SSGCount,
     })
   }
 
+  await copyStaticQueriesToEngine({
+    engineTemplatePaths,
+    staticQueriesByTemplate: store.getState().staticQueriesByTemplate,
+    components: store.getState().components,
+  })
+
   if (!(_CFLAGS_.GATSBY_MAJOR === `5` && process.env.GATSBY_SLICES)) {
     if (process.send && shouldGenerateEngines()) {
       await waitMaterializePageMode
@@ -577,29 +600,9 @@ module.exports = async function build(
     }
   }
 
-  {
-    let SSGCount = 0
-    let DSGCount = 0
-    let SSRCount = 0
-    for (const page of store.getState().pages.values()) {
-      if (page.mode === `SSR`) {
-        SSRCount++
-      } else if (page.mode === `DSG`) {
-        DSGCount++
-      } else {
-        SSGCount++
-      }
-    }
-
-    telemetry.addSiteMeasurement(`BUILD_END`, {
-      pagesCount: toRegenerate.length, // number of html files that will be written
-      totalPagesCount: store.getState().pages.size, // total number of pages
-      SSRCount,
-      DSGCount,
-      SSGCount,
-    })
-  }
-
+  telemetry.addSiteMeasurement(`BUILD_END`, {
+    totalPagesCount: store.getState().pages.size, // total number of pages
+  })
   const postBuildActivityTimer = report.activityTimer(`onPostBuild`, {
     parentSpan: buildSpan,
   })

packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts

@@ -10,7 +10,6 @@ import {
   getScriptsAndStylesForTemplate,
   readWebpackStats,
 } from "../client-assets-for-template"
-import { writeStaticQueryContext } from "../static-query-utils"
 import { IGatsbyState } from "../../redux/types"
 import { store } from "../../redux"
 
@@ -20,27 +19,45 @@ const extensions = [`.mjs`, `.js`, `.json`, `.node`, `.ts`, `.tsx`]
 const outputDir = path.join(process.cwd(), `.cache`, `page-ssr`)
 const cacheLocation = path.join(process.cwd(), `.cache`, `webpack`, `page-ssr`)
 
-export async function writeQueryContext({
-  staticQueriesByTemplate,
+export async function copyStaticQueriesToEngine({
+  engineTemplatePaths,
   components,
+  staticQueriesByTemplate,
 }: {
-  staticQueriesByTemplate: IGatsbyState["staticQueriesByTemplate"]
+  engineTemplatePaths: Set<string>
   components: IGatsbyState["components"]
+  staticQueriesByTemplate: IGatsbyState["staticQueriesByTemplate"]
 }): Promise<void> {
-  const waitingForWrites: Array<Promise<unknown>> = []
-  for (const pageTemplate of components.values()) {
-    const staticQueryHashes =
-      staticQueriesByTemplate.get(pageTemplate.componentPath) || []
+  const staticQueriesToCopy = new Set<string>()
+
+  for (const component of components.values()) {
+    // figuring out needed slices for each pages using componentPath is not straightforward
+    // so for now we just collect static queries for all slices + engine templates
+    if (component.isSlice || engineTemplatePaths.has(component.componentPath)) {
+      const staticQueryHashes =
+        staticQueriesByTemplate.get(component.componentPath) || []
+
+      for (const hash of staticQueryHashes) {
+        staticQueriesToCopy.add(hash)
+      }
+    }
+  }
+
+  const sourceDir = path.join(process.cwd(), `public`, `page-data`, `sq`, `d`)
+  const destDir = path.join(outputDir, `sq`)
+
+  await fs.ensureDir(destDir)
+  await fs.emptyDir(destDir)
+
+  const promisesToAwait: Array<Promise<void>> = []
+  for (const hash of staticQueriesToCopy) {
+    const sourcePath = path.join(sourceDir, `${hash}.json`)
+    const destPath = path.join(destDir, `${hash}.json`)
 
-    waitingForWrites.push(
-      writeStaticQueryContext(
-        staticQueryHashes,
-        pageTemplate.componentChunkName
-      )
-    )
+    promisesToAwait.push(fs.copy(sourcePath, destPath))
   }
 
-  return Promise.all(waitingForWrites).then(() => {})
+  await Promise.all(promisesToAwait)
 }
 
 export async function createPageSSRBundle({

packages/gatsby/src/utils/page-ssr-module/entry.ts

@@ -314,16 +314,10 @@ export async function renderPageData({
     }
   }
 }
-
-const readStaticQueryContext = async (
-  templatePath: string
+const readStaticQuery = async (
+  staticQueryHash: string
 ): Promise<Record<string, { data: unknown }>> => {
-  const filePath = path.join(
-    __dirname,
-    `sq-context`,
-    templatePath,
-    `sq-context.json`
-  )
+  const filePath = path.join(__dirname, `sq`, `${staticQueryHash}.json`)
   const rawSQContext = await fs.readFile(filePath, `utf-8`)
 
   return JSON.parse(rawSQContext)
@@ -399,20 +393,19 @@ export async function renderHTML({
         readStaticQueryContextActivity.start()
       }
 
-      const uniqueUsedComponentChunkNames = [data.page.componentChunkName]
+      const staticQueryHashes = new Set<string>(pageData.staticQueryHashes)
       for (const singleSliceData of Object.values(sliceData)) {
-        if (
-          singleSliceData.componentChunkName &&
-          !uniqueUsedComponentChunkNames.includes(
-            singleSliceData.componentChunkName
-          )
-        ) {
-          uniqueUsedComponentChunkNames.push(singleSliceData.componentChunkName)
+        for (const staticQueryHash of singleSliceData.staticQueryHashes) {
+          staticQueryHashes.add(staticQueryHash)
         }
       }
 
       const contextsToMerge = await Promise.all(
-        uniqueUsedComponentChunkNames.map(readStaticQueryContext)
+        Array.from(staticQueryHashes).map(async staticQueryHash => {
+          return {
+            [staticQueryHash]: await readStaticQuery(staticQueryHash),
+          }
+        })
       )
 
       staticQueryContext = Object.assign({}, ...contextsToMerge)

packages/gatsby/src/utils/static-query-utils.ts

@@ -5,8 +5,6 @@ const { join } = path.posix
 import type { IScriptsAndStyles } from "./client-assets-for-template"
 import { IPageDataWithQueryResult } from "./page-data"
 
-const outputDir = path.join(process.cwd(), `.cache`, `page-ssr`)
-
 export const getStaticQueryPath = (hash: string): string =>
   join(`page-data`, `sq`, `d`, `${hash}.json`)
 
@@ -66,24 +64,3 @@ export const getStaticQueryContext = async (
 
   return { staticQueryContext }
 }
-
-export const writeStaticQueryContext = async (
-  staticQueryHashes: IPageDataWithQueryResult["staticQueryHashes"],
-  templatePath: string
-): Promise<{
-  staticQueryContext: IResourcesForTemplate["staticQueryContext"]
-}> => {
-  const outputFilePath = path.join(
-    outputDir,
-    `sq-context`,
-    templatePath,
-    `sq-context.json`
-  )
-
-  const { staticQueryContext } = await getStaticQueryContext(staticQueryHashes)
-
-  const stringifiedContext = JSON.stringify(staticQueryContext)
-  await fs.outputFile(outputFilePath, stringifiedContext)
-
-  return { staticQueryContext }
-}