feat: allow dsg/ssr renders without access to datastore if it's not required (#38974)

* fix: generate lmdb cdn path after state was reloaded from cache restoration

* feat: make lmdb datastore optional and lazy in engines

* use datastore exlucde and skip deploy cleanup temporarily

* fix lint

* build engines after page configs were prepared

* tmp: workaround local deploy problems with datastore exclussion

* set rawUrl on request and use it to extract origin for engine download in case url passed at build doesn't work

* no unhandled promise rejections, actually pass rawUrl

* make one of e2e variants use datastore exclussion, so base case is still tested

* revert adapters e2e test cleanup

* scripts rename

* findPageByPath renaming

* getGraphqlEngine renaming

(cherry picked from commit 884ecaf)

Author: pieh

e2e-tests/adapters/package.json

@@ -14,14 +14,14 @@
     "develop:debug": "start-server-and-test develop http://localhost:8000 'npm run cy:open -- --config baseUrl=http://localhost:8000'",
     "ssat:debug": "start-server-and-test serve http://localhost:9000 cy:open",
     "test:template": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX node ../../scripts/cypress-run-with-conditional-record-flag.js --browser chrome --e2e --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX",
-    "test:template:debug": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX npm run cy:open -- --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX",
+    "test:template:debug": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--} / excludeDatastoreFromBundle:${GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE:-false}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX npm run cy:open -- --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX",
     "test:debug": "npm-run-all -s build:debug ssat:debug",
     "test:netlify": "cross-env TRAILING_SLASH=always node scripts/deploy-and-run/netlify.mjs test:template",
     "test:smoke": "node smoke-test.mjs",
     "test:netlify:debug": "cross-env TRAILING_SLASH=always node scripts/deploy-and-run/netlify.mjs test:template:debug",
-    "test:netlify:prefix-never": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix node scripts/deploy-and-run/netlify.mjs test:template",
-    "test:netlify:prefix-never:debug": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix node scripts/deploy-and-run/netlify.mjs test:template:debug",
-    "test": "npm-run-all -c -s test:netlify test:netlify:prefix-never"
+    "test:netlify:non-defaults-variant": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE=true node scripts/deploy-and-run/netlify.mjs test:template",
+    "test:netlify:non-defaults-variant:debug": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE=true node scripts/deploy-and-run/netlify.mjs test:template:debug",
+    "test": "npm-run-all -c -s test:netlify test:netlify:non-defaults-variant"
   },
   "dependencies": {
     "gatsby": "next",

packages/gatsby-adapter-netlify/src/lambda-handler.ts

@@ -176,11 +176,13 @@ const createRequestObject = ({ event, context }) => {
     multiValueHeaders = {},
     body,
     isBase64Encoded,
+    rawUrl
   } = event
   const newStream = new Stream.Readable()
   const req = Object.assign(newStream, http.IncomingMessage.prototype)
   req.url = path
   req.originalUrl = req.url
+  req.rawUrl = rawUrl
   req.query = queryStringParameters
   req.multiValueQuery = multiValueQueryStringParameters
   req.method = httpMethod

packages/gatsby/index.d.ts

@@ -1869,6 +1869,10 @@ export interface GatsbyFunctionRequest<ReqBody = any> extends IncomingMessage {
    * Object of `cookies` from header
    */
   cookies: Record<string, string>
+  /**
+   * Optional field to store the full raw URL by adapters
+   */
+  rawUrl?: string
 }
 
 export interface GatsbyFunctionBodyParserCommonMiddlewareConfig {

packages/gatsby/src/commands/build.ts

@@ -191,37 +191,6 @@ module.exports = async function build(
     buildActivityTimer.end()
   }
 
-  if (shouldGenerateEngines()) {
-    const state = store.getState()
-    const buildActivityTimer = report.activityTimer(
-      `Building Rendering Engines`,
-      { parentSpan: buildSpan }
-    )
-    try {
-      buildActivityTimer.start()
-      // bundle graphql-engine
-      engineBundlingPromises.push(
-        createGraphqlEngineBundle(program.directory, report, program.verbose)
-      )
-
-      engineBundlingPromises.push(
-        createPageSSRBundle({
-          rootDir: program.directory,
-          components: state.components,
-          staticQueriesByTemplate: state.staticQueriesByTemplate,
-          webpackCompilationHash: webpackCompilationHash as string, // we set webpackCompilationHash above
-          reporter: report,
-          isVerbose: program.verbose,
-        })
-      )
-      await Promise.all(engineBundlingPromises)
-    } catch (err) {
-      reporter.panic(err)
-    } finally {
-      buildActivityTimer.end()
-    }
-  }
-
   const buildSSRBundleActivityProgress = report.activityTimer(
     `Building HTML renderer`,
     { parentSpan: buildSpan }
@@ -295,6 +264,35 @@ module.exports = async function build(
   }
 
   if (shouldGenerateEngines()) {
+    const state = store.getState()
+    const buildActivityTimer = report.activityTimer(
+      `Building Rendering Engines`,
+      { parentSpan: buildSpan }
+    )
+    try {
+      buildActivityTimer.start()
+      // bundle graphql-engine
+      engineBundlingPromises.push(
+        createGraphqlEngineBundle(program.directory, report, program.verbose)
+      )
+
+      engineBundlingPromises.push(
+        createPageSSRBundle({
+          rootDir: program.directory,
+          components: state.components,
+          staticQueriesByTemplate: state.staticQueriesByTemplate,
+          webpackCompilationHash: webpackCompilationHash as string, // we set webpackCompilationHash above
+          reporter: report,
+          isVerbose: program.verbose,
+        })
+      )
+      await Promise.all(engineBundlingPromises)
+    } catch (err) {
+      reporter.panic(err)
+    } finally {
+      buildActivityTimer.end()
+    }
+
     await validateEnginesWithActivity(program.directory, buildSpan)
   }
 

packages/gatsby/src/commands/serve.ts

@@ -202,7 +202,7 @@ module.exports = async (program: IServeProgram): Promise<void> => {
     try {
       const { GraphQLEngine } =
         require(graphqlEnginePath) as typeof import("../schema/graphql-engine/entry")
-      const { getData, renderPageData, renderHTML } =
+      const { getData, renderPageData, renderHTML, findEnginePageByPath } =
         require(pageSSRModule) as typeof import("../utils/page-ssr-module/entry")
       const graphqlEngine = new GraphQLEngine({
         dbPath: path.posix.join(
@@ -222,7 +222,7 @@ module.exports = async (program: IServeProgram): Promise<void> => {
           }
 
           const potentialPagePath = reverseFixedPagePath(requestedPagePath)
-          const page = graphqlEngine.findPageByPath(potentialPagePath)
+          const page = findEnginePageByPath(potentialPagePath)
 
           if (page && (page.mode === `DSG` || page.mode === `SSR`)) {
             const requestActivity = report.phantomActivity(
@@ -272,7 +272,7 @@ module.exports = async (program: IServeProgram): Promise<void> => {
       router.use(async (req, res, next) => {
         if (req.accepts(`html`)) {
           const potentialPagePath = req.path
-          const page = graphqlEngine.findPageByPath(potentialPagePath)
+          const page = findEnginePageByPath(potentialPagePath)
           if (page && (page.mode === `DSG` || page.mode === `SSR`)) {
             const requestActivity = report.phantomActivity(
               `request for "${req.path}"`

packages/gatsby/src/schema/graphql-engine/entry.ts

@@ -195,6 +195,9 @@ export class GraphQLEngine {
     }
   }
 
+  /**
+   * @deprecated use findEnginePageByPath exported from page-ssr module instead
+   */
   public findPageByPath(pathName: string): IGatsbyPage | undefined {
     // adapter so `findPageByPath` use SitePage nodes in datastore
     // instead of `pages` redux slice

packages/gatsby/src/utils/adapter/get-route-path.ts

@@ -11,7 +11,9 @@ function maybeDropNamedPartOfWildcard(
   return path.replace(/\*.+$/, `*`)
 }
 
-export function getRoutePathFromPage(page: IGatsbyPage): string {
+export function getRoutePathFromPage(
+  page: Pick<IGatsbyPage, "path" | "matchPath">
+): string {
   return maybeDropNamedPartOfWildcard(page.matchPath) ?? page.path
 }
 

packages/gatsby/src/utils/adapter/manager.ts

@@ -26,7 +26,7 @@ import { getPageMode } from "../page-mode"
 import { getStaticQueryPath } from "../static-query-utils"
 import { getAdapterInit } from "./init"
 import {
-  LmdbOnCdnPath,
+  getLmdbOnCdnPath,
   shouldBundleDatastore,
   shouldGenerateEngines,
 } from "../engines-helpers"
@@ -192,7 +192,7 @@ export async function initAdapterManager(): Promise<IAdapterManager> {
       }
 
       // handle lmdb file
-      const mdbInPublicPath = `public/${LmdbOnCdnPath}`
+      const mdbInPublicPath = `public/${getLmdbOnCdnPath()}`
       if (!shouldBundleDatastore()) {
         const mdbPath = getDefaultDbPath() + `/data.mdb`
         copy(mdbPath, mdbInPublicPath)

packages/gatsby/src/utils/engines-helpers.ts

@@ -32,7 +32,7 @@ function getCDNObfuscatedPath(path: string): string {
   return `${store.getState().status.cdnObfuscatedPrefix}-${path}`
 }
 
-export const LmdbOnCdnPath = getCDNObfuscatedPath(`data.mdb`)
+export const getLmdbOnCdnPath = (): string => getCDNObfuscatedPath(`data.mdb`)
 
 export interface IPlatformAndArch {
   platform: string

packages/gatsby/src/utils/get-server-data.ts

@@ -23,7 +23,7 @@ export async function getServerData(
   req:
     | Partial<Pick<Request, "query" | "method" | "url" | "headers">>
     | undefined,
-  page: IGatsbyPage,
+  page: Pick<IGatsbyPage, "path" | "matchPath" | "context">,
   pagePath: string,
   mod: IModuleWithServerData | undefined
 ): Promise<IServerData> {

packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts

@@ -4,15 +4,16 @@ import webpack from "webpack"
 import mod from "module"
 import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging"
 import reporter from "gatsby-cli/lib/reporter"
-import type { ITemplateDetails } from "./entry"
+import type { EnginePage, ITemplateDetails } from "./entry"
 
 import {
   getScriptsAndStylesForTemplate,
   readWebpackStats,
 } from "../client-assets-for-template"
 import { IGatsbyState } from "../../redux/types"
 import { store } from "../../redux"
-import { LmdbOnCdnPath, shouldBundleDatastore } from "../engines-helpers"
+import { getLmdbOnCdnPath, shouldBundleDatastore } from "../engines-helpers"
+import { getPageMode } from "../page-mode"
 
 type Reporter = typeof reporter
 
@@ -112,6 +113,25 @@ export async function createPageSSRBundle({
     }
   }
 
+  const pagesIterable: Array<[string, EnginePage]> = []
+  for (const [pagePath, page] of state.pages) {
+    const mode = getPageMode(page, state)
+    if (mode !== `SSG`) {
+      pagesIterable.push([
+        pagePath,
+        {
+          componentChunkName: page.componentChunkName,
+          componentPath: page.componentPath,
+          context: page.context,
+          matchPath: page.matchPath,
+          mode,
+          path: page.path,
+          slices: page.slices,
+        },
+      ])
+    }
+  }
+
   const compiler = webpack({
     name: `Page Engine`,
     mode: `none`,
@@ -193,6 +213,7 @@ export async function createPageSSRBundle({
         INLINED_TEMPLATE_TO_DETAILS: JSON.stringify(toInline),
         INLINED_HEADERS_CONFIG: JSON.stringify(state.config.headers),
         WEBPACK_COMPILATION_HASH: JSON.stringify(webpackCompilationHash),
+        GATSBY_PAGES: JSON.stringify(pagesIterable),
         GATSBY_SLICES: JSON.stringify(slicesStateObject),
         GATSBY_SLICES_BY_TEMPLATE: JSON.stringify(slicesByTemplateStateObject),
         GATSBY_SLICES_SCRIPT: JSON.stringify(
@@ -248,9 +269,11 @@ export async function createPageSSRBundle({
   functionCode = functionCode
     .replaceAll(
       `%CDN_DATASTORE_PATH%`,
-      shouldBundleDatastore()
-        ? ``
-        : `${state.adapter.config.deployURL ?? ``}/${LmdbOnCdnPath}`
+      shouldBundleDatastore() ? `` : getLmdbOnCdnPath()
+    )
+    .replaceAll(
+      `%CDN_DATASTORE_ORIGIN%`,
+      shouldBundleDatastore() ? `` : state.adapter.config.deployURL ?? ``
     )
     .replaceAll(`%PATH_PREFIX%`, pathPrefix)
     .replaceAll(