fix(gatsby-adapter-netlify): support monorepos (#39005)

pieh · web-flow · commit 1a56c3433a42 · 2024-07-11T19:10:46.000+02:00
* test: run e2e adapters in monorepo setup

* fix(gatsby-adapter-netlify): ensure we set includedFilesBasePath so monorepo cases will bundle all required files correctly

* fix(gatsby): don't rely on process.cwd() in lambda for ssr/dsg, instead infer root of 'gatsby' through relative path from entry module and set process.cwd() based on that so rest of utils relying on process.cwd() continue to work in monorepo scenarios

* chore: minor cleanup

* chore: bump caniuse-lite
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -134,6 +134,9 @@ commands:
       slices:
         type: boolean
         default: true # allow disabling it later when setting up partial hydration tests
+      pre_gatsby_dev_command:
+        type: string
+        default: ""
     steps:
       - checkout
       # In case of failure, add these steps again. Cache probably got deleted
@@ -164,7 +167,7 @@ commands:
           command: yarn global add gatsby-dev-cli@next --ignore-engines
       - run:
           name: Run tests (using defaults)
-          command: ./scripts/e2e-test.sh "<< parameters.test_path >>" "<< parameters.test_command >>"
+          command: ./scripts/e2e-test.sh "<< parameters.test_path >>" "<< parameters.test_command >>" "<< parameters.pre_gatsby_dev_command >>"
 
 version: 2.1
 
@@ -507,6 +510,19 @@ jobs:
       - store_test_results:
           path: e2e-tests/adapters/cypress/results
 
+  e2e_tests_adapters_monorepo:
+    <<: *e2e-executor
+    docker:
+      - image: cypress/browsers:node-18.16.1-chrome-114.0.5735.133-1-ff-114.0.2-edge-114.0.1823.51-1
+    steps:
+      - run: echo 'export CYPRESS_RECORD_KEY="${CY_CLOUD_ADAPTERS}"' >> "$BASH_ENV"
+      - e2e-test:
+          test_path: e2e-tests/adapters
+          test_command: cd workspace; gatsby-dev --force-install --scan-once; cd ..; yarn test
+          pre_gatsby_dev_command: ./make-monorepo.sh
+      - store_test_results:
+          path: e2e-tests/adapters/cypress/results
+
   starters_validate:
     executor: node
     steps:
@@ -708,6 +724,8 @@ workflows:
           <<: *e2e-test-workflow
       - e2e_tests_adapters:
           <<: *e2e-test-workflow
+      - e2e_tests_adapters_monorepo:
+          <<: *e2e-test-workflow
       - e2e_tests_development_runtime_with_react_18:
           <<: *e2e-test-workflow
       - e2e_tests_production_runtime_with_react_18:
diff --git a/e2e-tests/adapters/make-monorepo.sh b/e2e-tests/adapters/make-monorepo.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# this script will make checked out files move around and make git working directory dirty
+# this is primarly for setting up monorepo structure for e2e tests that run in CI to avoid
+# having to maintain additional fixture as we want to test all the same things as in single repo
+# case
+
+# move all items in the current directory to a new directory called workspace
+rm -rf workspace
+items=(*)
+mkdir workspace
+mv ${items[*]} workspace
+
+# create root package.json and mark workspace directory as a npm/yarn workspace
+echo '{ "workspaces": ["workspace"], "scripts": { "test": "EXTRA_NTL_CLI_ARGS=\"--filter=workspace\" E2E_MONOREPO=\"true\" npm run test -w workspace" }, "private": true }' > package.json
+
+# update netlify.toml build command and publish dir
+sed -i.bak -e 's/npm run build/npm run build -w workspace/g' -e 's/public/workspace\/public/g' workspace/netlify.toml
+
+# update workspace's package.json to have correct path to helper script after moving site fixture into subdirectory
+sed -i.bak -e 's/..\/..\/scripts\/cypress-run-with-conditional-record-flag.js/..\/..\/..\/scripts\/cypress-run-with-conditional-record-flag.js/g' workspace/package.json
+
+git init
+
diff --git a/e2e-tests/adapters/netlify.toml b/e2e-tests/adapters/netlify.toml
@@ -1,12 +1,13 @@
 [build]
-  command = "npm run build"
-  publish = "public/"
+command = "npm run build"
+publish = "public/"
+base = ""
 
 [build.environment]
-  NETLIFY_IMAGE_CDN = "true"
+NETLIFY_IMAGE_CDN = "true"
 
 [images]
-  remote_images = [
-    "https://images.unsplash.com/.*",
-    "https://www.gatsbyjs.com/.*",
-  ]
+remote_images = [
+  "https://images.unsplash.com/.*",
+  "https://www.gatsbyjs.com/.*",
+]
diff --git a/e2e-tests/adapters/package.json b/e2e-tests/adapters/package.json
@@ -13,8 +13,8 @@
     "cy:open": "cypress open --browser chrome --e2e",
     "develop:debug": "start-server-and-test develop http://localhost:8000 'npm run cy:open -- --config baseUrl=http://localhost:8000'",
     "ssat:debug": "start-server-and-test serve http://localhost:9000 cy:open",
-    "test:template": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX node ../../scripts/cypress-run-with-conditional-record-flag.js --browser chrome --e2e --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX",
-    "test:template:debug": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--} / excludeDatastoreFromBundle:${GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE:-false}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX npm run cy:open -- --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX",
+    "test:template": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--}  / excludeDatastoreFromBundle:${GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE:-false} / monorepo:${E2E_MONOREPO:-false}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX node ../../scripts/cypress-run-with-conditional-record-flag.js --browser chrome --e2e --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX",
+    "test:template:debug": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--} / excludeDatastoreFromBundle:${GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE:-false} / monorepo:${E2E_MONOREPO:-false}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX npm run cy:open -- --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX",
     "test:debug": "npm-run-all -s build:debug ssat:debug",
     "test:netlify": "cross-env TRAILING_SLASH=always node scripts/deploy-and-run/netlify.mjs test:template",
     "test:smoke": "node smoke-test.mjs",
@@ -38,7 +38,7 @@
     "dotenv": "^8.6.0",
     "execa": "^6.1.0",
     "gatsby-cypress": "^3.13.0",
-    "netlify-cli": "^17.9.0",
+    "netlify-cli": "^17.25.0",
     "npm-run-all": "^4.1.5",
     "start-server-and-test": "^2.0.3",
     "typescript": "^5.3.3"
diff --git a/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs b/e2e-tests/adapters/scripts/deploy-and-run/netlify.mjs
@@ -19,8 +19,16 @@ const npmScriptToRun = process.argv[2] || "test:netlify"
 await execa(`npm`, [`run`, `clean`], { stdio: `inherit` })
 
 const deployResults = await execa(
-  "ntl",
-  ["deploy", "--build", "--json", "--cwd=.", "--message", deployTitle],
+  "npx",
+  [
+    "ntl",
+    "deploy",
+    "--build",
+    "--json",
+    "--message",
+    deployTitle,
+    process.env.EXTRA_NTL_CLI_ARGS ?? "--cwd=.",
+  ],
   {
     reject: false,
   }
@@ -49,7 +57,8 @@ try {
 } finally {
   if (!process.env.GATSBY_TEST_SKIP_CLEANUP) {
     console.log(`Deleting project with deploy_id ${deployInfo.deploy_id}`)
-    const deleteResponse = await execa("ntl", [
+    const deleteResponse = await execa("npx", [
+      "ntl",
       "api",
       "deleteDeploy",
       "--data",
diff --git a/packages/gatsby-adapter-netlify/src/lambda-handler.ts b/packages/gatsby-adapter-netlify/src/lambda-handler.ts
@@ -61,6 +61,7 @@ export async function prepareFunction(
       includedFiles: fun.requiredFiles.map(file =>
         slash(file).replace(/\[/g, `*`).replace(/]/g, `*`)
       ),
+      includedFilesBasePath: process.cwd(),
       externalNodeModules: [`msgpackr-extract`],
     },
     version: 1,
diff --git a/packages/gatsby/src/utils/page-ssr-module/lambda.ts b/packages/gatsby/src/utils/page-ssr-module/lambda.ts
@@ -17,12 +17,22 @@ const cdnDatastorePath = `%CDN_DATASTORE_PATH%`
 const cdnDatastoreOrigin = `%CDN_DATASTORE_ORIGIN%`
 const PATH_PREFIX = `%PATH_PREFIX%`
 
+// this file should be in `.cache/page-ssr-module/lambda.js`
+// so getting `.cache` location should be one directory above
+// directory of this file
+const cacheDir = path.join(__dirname, `..`)
+
+// gatsby relies on process.cwd() a lot, so this ensures that CWD is set correctly
+// in relation to bundled files for lambda. In some scenarios those files are not in
+// expected relative locations to CWD, so here we are forcing setting CWD so the
+// relative paths are correct
+process.chdir(path.join(cacheDir, `..`))
+
 function setupFsWrapper(): string {
   // setup global._fsWrapper
   try {
     fs.accessSync(__filename, fs.constants.W_OK)
-    // TODO: this seems funky - not sure if this is correct way to handle this, so just marking TODO to revisit this
-    return path.join(__dirname, `..`, `data`, `datastore`)
+    return path.join(cacheDir, `data`, `datastore`)
   } catch (e) {
     // we are in a read-only filesystem, so we need to use a temp dir
 
@@ -31,9 +41,6 @@ function setupFsWrapper(): string {
 
     global.__GATSBY.root = TEMP_DIR
 
-    // TODO: don't hardcode this
-    const cacheDir = `${process.cwd()}/.cache`
-
     // we need to rewrite fs
     const rewrites = [
       [path.join(cacheDir, `caches`), path.join(TEMP_CACHE_DIR, `caches`)],
diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh
@@ -3,6 +3,7 @@ set -e # bail on errors
 
 SRC_PATH=$1
 CUSTOM_COMMAND="${2:-yarn test}"
+PRE_GATSBY_DEV_COMMAND="${3:-}"
 eval GATSBY_PATH=${CIRCLE_WORKING_DIRECTORY:-../..}
 TMP_LOCATION=$(mktemp -d);
 
@@ -22,6 +23,11 @@ cp -Rv $GATSBY_PATH/scripts/. $TMP_LOCATION/scripts/
 # setting up child integration test link to gatsby packages
 cd "$TMP_TEST_LOCATION"
 
+if [[ $PRE_GATSBY_DEV_COMMAND != "" ]]; then
+  echo "Running pre-gatsby-dev command: $PRE_GATSBY_DEV_COMMAND"
+  sh -c "$PRE_GATSBY_DEV_COMMAND"
+fi
+
 gatsby-dev --set-path-to-repo "$GATSBY_PATH"
 gatsby-dev --force-install --scan-once  # Do not copy files, only install through npm, like our users would
 if test -f "./node_modules/.bin/gatsby"; then
diff --git a/yarn.lock b/yarn.lock
@@ -7562,9 +7562,9 @@ caniuse-api@^3.0.0:
     lodash.uniq "^4.5.0"
 
 caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001538, caniuse-lite@^1.0.30001565:
-  version "1.0.30001572"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001572.tgz#1ccf7dc92d2ee2f92ed3a54e11b7b4a3041acfa0"
-  integrity sha512-1Pbh5FLmn5y4+QhNyJE9j3/7dK44dGB83/ZMjv/qJk86TvDbjk0LosiZo0i0WB0Vx607qMX9jYrn1VLHCkN4rw==
+  version "1.0.30001641"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001641.tgz"
+  integrity sha512-Phv5thgl67bHYo1TtMY/MurjkHhV4EDaCosezRXgZ8jzA/Ub+wjxAvbGvjoFENStinwi5kCyOYV3mi5tOGykwA==
 
 capital-case@^1.0.4:
   version "1.0.4"
