From ce93f8917037a80e626a48a9cfa0bf63a2404609 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 22 Dec 2023 09:16:21 +0100 Subject: [PATCH 1/3] feat: prefer github for adapters version manifest --- .../__tests__/get-latest-gatsby-files.ts | 31 ++++++++++++++-- .../src/utils/get-latest-gatsby-files.ts | 36 +++++++++++++++---- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts b/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts index 0352115f24507..8019bb7bf1606 100644 --- a/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts +++ b/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts @@ -108,13 +108,40 @@ describe(`default behavior: has network connectivity`, () => { }) describe(`getLatestAdapters`, () => { - beforeEach(() => { + it(`loads .js modules (prefers github)`, async () => { axios.get.mockResolvedValueOnce({ data: latestAdaptersMarker }) + const data = await getLatestAdapters() + + expect(axios.get).toHaveBeenCalledWith( + expect.stringContaining(`raw.githubusercontent.com`), + expect.any(Object) + ) + + expect(axios.get).not.toHaveBeenCalledWith( + expect.stringContaining(`unpkg.com`), + expect.any(Object) + ) + + expect(fs.writeFile).toHaveBeenCalledWith( + expect.stringContaining(`latest-adapters.js`), + latestAdaptersMarker, + expect.any(String) + ) + + expect(data).toEqual(mockAdaptersManifest) }) - it(`loads .js modules`, async () => { + it(`loads .js modules (fallbacks to unkpg of github fails)`, async () => { + axios.get.mockRejectedValueOnce(new Error(`does not matter`)) + axios.get.mockResolvedValueOnce({ data: latestAdaptersMarker }) + const data = await getLatestAdapters() + expect(axios.get).toHaveBeenCalledWith( + expect.stringContaining(`raw.githubusercontent.com`), + expect.any(Object) + ) + expect(axios.get).toHaveBeenCalledWith( expect.stringContaining(`unpkg.com`), expect.any(Object) diff --git a/packages/gatsby/src/utils/get-latest-gatsby-files.ts b/packages/gatsby/src/utils/get-latest-gatsby-files.ts index a2ff122a00a06..6ac5af8c93d9c 100644 --- a/packages/gatsby/src/utils/get-latest-gatsby-files.ts +++ b/packages/gatsby/src/utils/get-latest-gatsby-files.ts @@ -6,6 +6,7 @@ import { preferDefault } from "../bootstrap/prefer-default" const ROOT = path.join(__dirname, `..`, `..`) const UNPKG_ROOT = `https://unpkg.com/gatsby/` +const GITHUB_ROOT = `https://raw.githubusercontent.com/gatsbyjs/gatsby/master/packages/gatsby/` const FILE_NAMES = { APIS: `apis.json`, @@ -23,29 +24,49 @@ export interface IAPIResponse { ssr: Record } +const _fetchFile = async (root: string, fileName: string): Promise => { + try { + const { data } = await axios.get(`${root}${fileName}`, { + timeout: 5000, + }) + return data + } catch (e) { + return null + } +} + const _getFile = async ({ fileName, outputFileName, defaultReturn, + tryGithubFirst, }: { fileName: string outputFileName: string defaultReturn: T + tryGithubFirst?: boolean }): Promise => { let fileToUse = path.join(ROOT, fileName) - try { - const { data } = await axios.get(`${UNPKG_ROOT}${fileName}`, { - timeout: 5000, - }) + let fetchedData = null + if (tryGithubFirst) { + fetchedData = await _fetchFile(GITHUB_ROOT, fileName) + } + if (!fetchedData) { + fetchedData = await _fetchFile(UNPKG_ROOT, fileName) + } + + if (fetchedData) { await fs.writeFile( outputFileName, - typeof data === `string` ? data : JSON.stringify(data, null, 2), + typeof fetchedData === `string` + ? fetchedData + : JSON.stringify(fetchedData, null, 2), `utf8` ) fileToUse = outputFileName - } catch (e) { + } else { // if file was previously cached, use it if (await fs.pathExists(outputFileName)) { fileToUse = outputFileName @@ -84,4 +105,7 @@ export const getLatestAdapters = async (): Promise< fileName: FILE_NAMES.ADAPTERS, outputFileName: OUTPUT_FILES.ADAPTERS, defaultReturn: [], + // trying github first for adapters manifest to be able to faster make changes to version manifest + // as publishing latest version of gatsby package takes more time + tryGithubFirst: true, }) From 86e60e02c4211ace07c8f37742b35df33e2bb31b Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 22 Dec 2023 17:17:23 +0100 Subject: [PATCH 2/3] fix(gatsby): more robust adapter autoinstallation handling to prevent broken deploys --- .../src/utils/adapter/__tests__/init.ts | 542 ++++++++++++++++++ packages/gatsby/src/utils/adapter/init.ts | 345 +++++++---- packages/gatsby/src/utils/adapter/types.ts | 4 + 3 files changed, 767 insertions(+), 124 deletions(-) create mode 100644 packages/gatsby/src/utils/adapter/__tests__/init.ts diff --git a/packages/gatsby/src/utils/adapter/__tests__/init.ts b/packages/gatsby/src/utils/adapter/__tests__/init.ts new file mode 100644 index 0000000000000..301d352f5e0cb --- /dev/null +++ b/packages/gatsby/src/utils/adapter/__tests__/init.ts @@ -0,0 +1,542 @@ +import semverMaxSatisfying from "semver/ranges/max-satisfying" + +import { getAdapterInit, getAdaptersCacheDir } from "../init" +import { AdapterInit, IAdapter, IAdapterManifestEntry } from "../types" +import execa from "execa" + +let mockAdaptersManifest: Array = [] + +let mockLogs: Array<{ + level: string + args: Array + spinnerArgs?: Array +}> = [] + +function getLogsForSnapshot(): string { + return mockLogs + .map( + log => + `${log.level.padEnd(13)} ${[...log.args, ...(log.spinnerArgs ?? [])] + .map(arg => JSON.stringify(arg)) + .join(` `)}` + ) + .join(`\n`) +} + +jest.mock(`gatsby-cli/lib/reporter`, () => { + return { + panic: jest.fn((...args) => { + mockLogs.push({ level: `panic`, args }) + }), + verbose: jest.fn((...args) => { + mockLogs.push({ level: `verbose`, args }) + }), + warn: jest.fn((...args) => { + mockLogs.push({ level: `warn`, args }) + }), + info: jest.fn((...args) => { + mockLogs.push({ level: `info`, args }) + }), + activityTimer: jest.fn((...spinnerArgs) => { + return { + start: jest.fn((...args) => + mockLogs.push({ level: `spinner-start`, args, spinnerArgs }) + ), + end: jest.fn((...args) => { + mockLogs.push({ level: `spinner-end`, args, spinnerArgs }) + }), + panic: jest.fn((...args) => { + mockLogs.push({ level: `spinner-panic`, args, spinnerArgs }) + }), + } + }), + } +}) + +jest.mock(`../../get-latest-gatsby-files`, () => { + return { + getLatestAdapters: jest.fn(() => mockAdaptersManifest), + } +}) + +interface IMockedAdapterPackage { + version: string + init: AdapterInit +} + +let mockInstalledInSiteAdapter: IMockedAdapterPackage | undefined = undefined +let mockInstalledInCacheAdapter: IMockedAdapterPackage | undefined = undefined + +const mockAdaptersCacheDir = getAdaptersCacheDir() +jest.mock(`gatsby-core-utils/create-require-from-path`, () => { + return { + createRequireFromPath: jest.fn((path: string) => { + let mockPackage: IMockedAdapterPackage | undefined + let prefix: string | undefined + + if (path === `${process.cwd()}/:internal:`) { + mockPackage = mockInstalledInSiteAdapter + prefix = `site` + } else if (path === `${mockAdaptersCacheDir}/:internal:`) { + mockPackage = mockInstalledInCacheAdapter + prefix = `cache` + } + + // checking if installed in site + const siteRequire = (mod: string): any => { + if (mockPackage) { + if (mod === `gatsby-adapter-test/package.json`) { + return { + version: mockPackage.version, + } + } + if (mod === `gatsby-adapter-test`) { + return mockPackage.init + } + } + throw new Error(`Module not found`) + } + + siteRequire.resolve = (mod: string): string => `${prefix}/${mod}` + + return siteRequire + }), + } +}) + +const getMockedPackage = ( + versionRange: string +): IMockedAdapterPackage | undefined => { + const version = semverMaxSatisfying( + [`1.0.0`, `1.0.1`, `1.0.2`, `1.0.3`, `1.0.4`], + versionRange + ) + if (version) { + return { + version, + init: (): IAdapter => { + return { + name: `gatsby-adapter-test@${version}`, + adapt: (): void => {}, + } + }, + } + } else { + return undefined + } +} + +jest.mock(`execa`, () => + jest.fn((command, args) => { + if (command === `npm`) { + const [, range] = args + .find(arg => arg.includes(`gatsby-adapter-test`)) + .split(`@`) + + // set mock adapter as installed in cache + mockInstalledInCacheAdapter = getMockedPackage(range) + return + } + throw new Error(`not expected execa command: "${command}`) + }) +) + +const mockSiteAdapterModule = jest.fn(() => { + if (mockInstalledInSiteAdapter) { + return mockInstalledInSiteAdapter.init + } + + throw new Error(`Module not found`) +}) + +jest.mock(`site/gatsby-adapter-test`, () => mockSiteAdapterModule(), { + virtual: true, +}) + +const mockCacheAdapterModule = jest.fn(() => { + if (mockInstalledInCacheAdapter) { + return mockInstalledInCacheAdapter.init + } + + throw new Error(`Module not found`) +}) + +jest.mock(`cache/gatsby-adapter-test`, () => mockCacheAdapterModule(), { + virtual: true, +}) + +// note this is used only if used didn't explicitly set adapter in gatsby-config - zero-conf mode +describe(`getAdapterInit`, () => { + beforeEach(() => { + jest.resetModules() + mockLogs = [] + }) + + it(`no matching adapter modules for current environment skips providing any adapter`, async () => { + mockAdaptersManifest = [ + { + name: `Test`, + // this entry is not eligible for current environment + test: (): boolean => false, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + expect(await getAdapterInit(`5.11.0`)).toBeUndefined() + expect(getLogsForSnapshot()).toMatchInlineSnapshot( + `"verbose \\"No adapter was found for the current environment. Skipping adapter initialization.\\""` + ) + }) + + describe(`matching adapter module for current environment`, () => { + beforeEach(() => { + execa.mockClear() + mockInstalledInSiteAdapter = undefined + mockInstalledInCacheAdapter = undefined + delete process.env.GATSBY_CONTINUE_BUILD_ON_ADAPTER_MISMATCH + }) + + it(`panics if no matching adapter version for used gatsby version by default`, async () => { + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + + await getAdapterInit(`6.0.0`) + // panic fails the build + expect(mockLogs.find(log => log.level === `panic`)).toBeTruthy() + expect(getLogsForSnapshot()).toMatchInlineSnapshot( + `"panic \\"No version of Test adapter is compatible with your current Gatsby version 6.0.0./n/nZero-configuration deployment failed to avoid potentially broken deployment./nIf you want build to continue despite above problem/n - configure adapter manually in gatsby-config which will skip zero-configuration deployment attempt/n - or set GATSBY_CONTINUE_BUILD_ON_MISSING_ADAPTER=true environment variable to continue build without an adapter.\\""` + ) + }) + + it(`continue the build without adapter if no matching adapter version for used gatsby version and GATSBY_CONTINUE_BUILD_ON_ADAPTER_MISMATCH is used`, async () => { + process.env.GATSBY_CONTINUE_BUILD_ON_ADAPTER_MISMATCH = `true` + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.0`, + moduleVersion: `^1.0.0`, + }, + ], + }, + ] + + expect(await getAdapterInit(`6.0.0`)).toBeUndefined() + expect(getLogsForSnapshot()).toMatchInlineSnapshot( + `"warn \\"No version of Test adapter is compatible with your current Gatsby version 6.0.0./n/nContinuing build using without using any adapter due to GATSBY_CONTINUE_BUILD_ON_MISSING_ADAPTER environment variable being set\\""` + ) + }) + + it(`automatically installs correct version of adapter `, async () => { + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + + const adapterInit = await getAdapterInit(`5.12.0`) + expect(adapterInit).not.toBeUndefined() + expect(adapterInit?.().name).toMatchInlineSnapshot( + `"gatsby-adapter-test@1.0.3"` + ) + expect(execa).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + "npm", + Array [ + "install", + "--no-progress", + "--no-audit", + "--no-fund", + "--loglevel", + "error", + "--color", + "always", + "--legacy-peer-deps", + "--save-exact", + "gatsby-adapter-test@>=1.0.0 <=1.0.3", + ], + Object { + "cwd": "/.cache/adapters", + "stderr": "inherit", + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": undefined, + }, + ], + } + `) + expect(getLogsForSnapshot()).toMatchInlineSnapshot(` + "spinner-start \\"Installing Test adapter (gatsby-adapter-test@>=1.0.0 <=1.0.3)\\" + spinner-end \\"Installing Test adapter (gatsby-adapter-test@>=1.0.0 <=1.0.3)\\" + info \\"If you plan on staying on this deployment platform, consider installing /\\"gatsby-adapter-test@>=1.0.0 <=1.0.3/\\" as a dependency in your project. This will give you faster and more robust installs.\\"" + `) + }) + + it(`panics if automatic installation of correct version of adapter fails`, async () => { + execa.mockImplementationOnce(() => { + throw new Error(`npm install failed`) + }) + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + + const adapterInit = await getAdapterInit(`5.12.0`) + expect(adapterInit).toBeUndefined() + expect(mockLogs.find(log => log.level === `spinner-panic`)).toBeTruthy() + expect(execa).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + "npm", + Array [ + "install", + "--no-progress", + "--no-audit", + "--no-fund", + "--loglevel", + "error", + "--color", + "always", + "--legacy-peer-deps", + "--save-exact", + "gatsby-adapter-test@>=1.0.0 <=1.0.3", + ], + Object { + "cwd": "/.cache/adapters", + "stderr": "inherit", + }, + ], + ], + "results": Array [ + Object { + "type": "throw", + "value": [Error: npm install failed], + }, + ], + } + `) + expect(getLogsForSnapshot()).toMatchInlineSnapshot(` + "spinner-start \\"Installing Test adapter (gatsby-adapter-test@>=1.0.0 <=1.0.3)\\" + spinner-panic \\"Could not install adapter /\\"gatsby-adapter-test@>=1.0.0 <=1.0.3/\\". Please install it yourself by adding it to your package.json's dependencies and try building your project again./n/nZero-configuration deployment failed to avoid potentially broken deployment./nIf you want build to continue despite above problem/n - configure adapter manually in gatsby-config which will skip zero-configuration deployment attempt/n - or set GATSBY_CONTINUE_BUILD_ON_MISSING_ADAPTER=true environment variable to continue build without an adapter.\\" \\"Installing Test adapter (gatsby-adapter-test@>=1.0.0 <=1.0.3)\\"" + `) + }) + + it(`reuses previously auto-installed adapter if compatible`, async () => { + mockInstalledInCacheAdapter = getMockedPackage(`^1.0.4`) + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + + const adapterInit = await getAdapterInit(`5.12.10`) + expect(adapterInit).not.toBeUndefined() + expect(adapterInit?.().name).toMatchInlineSnapshot( + `"gatsby-adapter-test@1.0.4"` + ) + expect(execa).not.toHaveBeenCalled() + expect(getLogsForSnapshot()).toMatchInlineSnapshot( + `"verbose \\"Using previously adapter previously installed by gatsby /\\"gatsby-adapter-test@1.0.4/\\"\\""` + ) + }) + + it(`ignores previously auto-installed adapter if not compatible and installs compatible one`, async () => { + mockInstalledInCacheAdapter = getMockedPackage(`>=1.0.0 <=1.0.3`) + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + + const adapterInit = await getAdapterInit(`5.12.10`) + expect(adapterInit).not.toBeUndefined() + expect(adapterInit?.().name).toMatchInlineSnapshot( + `"gatsby-adapter-test@1.0.4"` + ) + expect(execa).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + "npm", + Array [ + "install", + "--no-progress", + "--no-audit", + "--no-fund", + "--loglevel", + "error", + "--color", + "always", + "--legacy-peer-deps", + "--save-exact", + "gatsby-adapter-test@^1.0.4", + ], + Object { + "cwd": "/.cache/adapters", + "stderr": "inherit", + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": undefined, + }, + ], + } + `) + expect(getLogsForSnapshot()).toMatchInlineSnapshot(` + "verbose \\"Ignoring incompatible gatsby-adapter-test installed by gatsby in /\\".cache/adapters/\\" before. Used gatsby version /\\"5.12.10/\\" requires /\\"gatsby-adapter-test@^1.0.4/\\". Installed /\\"gatsby-adapter-test/\\" version: /\\"1.0.3/\\".\\" + spinner-start \\"Installing Test adapter (gatsby-adapter-test@^1.0.4)\\" + spinner-end \\"Installing Test adapter (gatsby-adapter-test@^1.0.4)\\" + info \\"If you plan on staying on this deployment platform, consider installing /\\"gatsby-adapter-test@^1.0.4/\\" as a dependency in your project. This will give you faster and more robust installs.\\"" + `) + }) + + it(`uses site's adapter dependency if it's compatible with current gatsby version`, async () => { + mockInstalledInSiteAdapter = getMockedPackage(`>=1.0.0 <=1.0.3`) + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + + const adapterInit = await getAdapterInit(`5.12.0`) + expect(adapterInit).not.toBeUndefined() + expect(adapterInit?.().name).toMatchInlineSnapshot( + `"gatsby-adapter-test@1.0.3"` + ) + expect(getLogsForSnapshot()).toMatchInlineSnapshot( + `"verbose \\"Using site's adapter dependency /\\"gatsby-adapter-test@1.0.3/\\"\\""` + ) + }) + + it(`skips using site's adapter dependency if it's not compatible with current gatsby versions and auto-installs compatible one`, async () => { + mockInstalledInSiteAdapter = getMockedPackage(`>=1.0.0 <=1.0.3`) + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `^5.12.10`, + moduleVersion: `^1.0.4`, + }, + { + gatsbyVersion: `>=5.0.0 <5.12.10`, + moduleVersion: `>=1.0.0 <=1.0.3`, + }, + ], + }, + ] + + const adapterInit = await getAdapterInit(`5.12.10`) + expect(adapterInit).not.toBeUndefined() + expect(adapterInit?.().name).toMatchInlineSnapshot( + `"gatsby-adapter-test@1.0.4"` + ) + expect(getLogsForSnapshot()).toMatchInlineSnapshot(` + "warn \\"Ignoring incompatible gatsby-adapter-test@1.0.3 installed by site. Used gatsby version /\\"5.12.10/\\" requires /\\"gatsby-adapter-test@^1.0.4/\\". Installed /\\"gatsby-adapter-test/\\" version: /\\"1.0.3/\\".\\" + spinner-start \\"Installing Test adapter (gatsby-adapter-test@^1.0.4)\\" + spinner-end \\"Installing Test adapter (gatsby-adapter-test@^1.0.4)\\" + info \\"If you plan on staying on this deployment platform, consider installing /\\"gatsby-adapter-test@^1.0.4/\\" as a dependency in your project. This will give you faster and more robust installs.\\"" + `) + }) + }) +}) diff --git a/packages/gatsby/src/utils/adapter/init.ts b/packages/gatsby/src/utils/adapter/init.ts index 236aa278af0a5..5d7aecd50edc9 100644 --- a/packages/gatsby/src/utils/adapter/init.ts +++ b/packages/gatsby/src/utils/adapter/init.ts @@ -4,13 +4,14 @@ import { createRequireFromPath } from "gatsby-core-utils/create-require-from-pat import { join } from "path" import { emptyDir, ensureDir, outputJson } from "fs-extra" import execa, { Options as ExecaOptions } from "execa" -import { version as gatsbyVersion } from "gatsby/package.json" +import { version as gatsbyVersionFromPackageJson } from "gatsby/package.json" import { satisfies } from "semver" import type { AdapterInit } from "./types" import { preferDefault } from "../../bootstrap/prefer-default" import { getLatestAdapters } from "../get-latest-gatsby-files" -const getAdaptersCacheDir = (): string => join(process.cwd(), `.cache/adapters`) +export const getAdaptersCacheDir = (): string => + join(process.cwd(), `.cache/adapters`) const createAdaptersCacheDir = async (): Promise => { await ensureDir(getAdaptersCacheDir()) @@ -28,165 +29,261 @@ const createAdaptersCacheDir = async (): Promise => { }) } -export async function getAdapterInit(): Promise { - // 1. Find the correct adapter and its details (e.g. version) - const latestAdapters = await getLatestAdapters() - const adapterToUse = latestAdapters.find(candidate => candidate.test()) - - if (!adapterToUse) { - reporter.verbose( - `No adapter was found for the current environment. Skipping adapter initialization.` - ) - return undefined - } - - const versionForCurrentGatsbyVersion = adapterToUse.versions.find(entry => - satisfies(gatsbyVersion, entry.gatsbyVersion, { includePrerelease: true }) - ) - - if (!versionForCurrentGatsbyVersion) { - reporter.verbose( - `The ${adapterToUse.name} adapter is not compatible with your current Gatsby version ${gatsbyVersion}.` - ) - return undefined - } +interface IAdapterToUse { + name: string + module: string + gatsbyVersion: string + moduleVersion: string +} - // 2. Check if the user has manually installed the adapter and try to resolve it from there +const tryLoadingAlreadyInstalledAdapter = async ({ + adapterToUse, + installLocation, + currentGatsbyVersion, +}: { + adapterToUse: IAdapterToUse + currentGatsbyVersion: string + installLocation: string +}): Promise< + | { + found: false + } + | ({ + found: true + installedVersion: string + } & ( + | { + compatible: false + incompatibilityReason: string + } + | { + compatible: true + loadedModule: AdapterInit + } + )) +> => { try { - const siteRequire = createRequireFromPath(`${process.cwd()}/:internal:`) - const adapterPackageJson = siteRequire( - `${adapterToUse.module}/package.json` + const locationRequire = createRequireFromPath( + `${installLocation}/:internal:` ) - const adapterGatsbyPeerDependency = _.get( - adapterPackageJson, - `peerDependencies.gatsby` + const adapterPackageJson = locationRequire( + `${adapterToUse.module}/package.json` ) - const moduleVersion = adapterPackageJson?.version + const adapterPackageVersion = adapterPackageJson?.version - // Check if the peerDependency of the adapter is compatible with the current Gatsby version + // Check if installed adapter version is compatible with the current Gatsby version based on the manifest if ( - adapterGatsbyPeerDependency && - !satisfies(gatsbyVersion, adapterGatsbyPeerDependency, { + !satisfies(adapterPackageVersion, adapterToUse.moduleVersion, { includePrerelease: true, }) ) { - reporter.warn( - `The ${adapterToUse.name} adapter is not compatible with your current Gatsby version ${gatsbyVersion} - It requires gatsby@${adapterGatsbyPeerDependency}` - ) - return undefined + return { + found: true, + compatible: false, + installedVersion: adapterPackageVersion, + incompatibilityReason: `Used gatsby version "${currentGatsbyVersion}" requires "${adapterToUse.module}@${adapterToUse.moduleVersion}". Installed "${adapterToUse.module}" version: "${adapterPackageVersion}".`, + } } - // Cross-check the adapter version with the version manifest and see if the adapter version is correct for the current Gatsby version - const isAdapterCompatible = satisfies( - moduleVersion, - versionForCurrentGatsbyVersion.moduleVersion, - { - includePrerelease: true, + const required = locationRequire.resolve(adapterToUse.module) + if (required) { + return { + found: true, + compatible: true, + installedVersion: adapterPackageVersion, + loadedModule: preferDefault( + preferDefault(await import(required)) + ) as AdapterInit, + } + } else { + return { + found: false, } + } + } catch (e) { + return { + found: false, + } + } +} + +const handleAdapterProblem = ( + message: string, + panicFn = reporter.panic +): never | undefined => { + if (!process.env.GATSBY_CONTINUE_BUILD_ON_ADAPTER_MISMATCH) { + panicFn( + `${message}\n\nZero-configuration deployment failed to avoid potentially broken deployment.\nIf you want build to continue despite above problems:\n - configure adapter manually in gatsby-config which will skip zero-configuration deployment attempt\n - or set GATSBY_CONTINUE_BUILD_ON_MISSING_ADAPTER=true environment variable to continue build without an adapter.` + ) + } else { + reporter.warn( + `${message}\n\nContinuing build using without using any adapter due to GATSBY_CONTINUE_BUILD_ON_MISSING_ADAPTER environment variable being set` ) + } + return undefined +} - if (!isAdapterCompatible) { - reporter.warn( - `${adapterToUse.module}@${moduleVersion} is not compatible with your current Gatsby version ${gatsbyVersion} - Install ${adapterToUse.module}@${versionForCurrentGatsbyVersion.moduleVersion} or later.` - ) +export async function getAdapterInit( + currentGatsbyVersion: string = gatsbyVersionFromPackageJson +): Promise { + // 0. Try to fetch the latest adapters manifest - if it fails, we continue with manifest packaged with current version of gatsby + const latestAdapters = await getLatestAdapters() - return undefined - } + // 1. Find adapter candidates that are compatible with the current environment + // we find all matching adapters in case package is renamed in the future and future gatsby versions will need different package than previous ones + const adapterEntry = latestAdapters.find(candidate => candidate.test()) - const required = siteRequire.resolve(adapterToUse.module) + if (!adapterEntry) { + reporter.verbose( + `No adapter was found for the current environment. Skipping adapter initialization.` + ) + return undefined + } - if (required) { - reporter.verbose( - `Reusing existing adapter ${adapterToUse.module} inside node_modules` - ) + // 2.From the manifest entry find one that supports current Gatsby version and identify it's version to use + // First matching one will be used. + let adapterToUse: IAdapterToUse | undefined = undefined - // TODO: double preferDefault is most ceirtainly wrong - figure it out - return preferDefault(preferDefault(await import(required))) as AdapterInit + for (const versionEntry of adapterEntry.versions) { + if ( + satisfies(currentGatsbyVersion, versionEntry.gatsbyVersion, { + includePrerelease: true, + }) + ) { + adapterToUse = { + name: adapterEntry.name, + module: versionEntry.module ?? adapterEntry.module, + gatsbyVersion: versionEntry.gatsbyVersion, + moduleVersion: versionEntry.moduleVersion, + } + break } - } catch (e) { - // no-op } - // 3. Check if a previous run has installed the correct adapter into .cache/adapters already and try to resolve it from there - try { - const adaptersRequire = createRequireFromPath( - `${getAdaptersCacheDir()}/:internal:` + if (!adapterToUse) { + return handleAdapterProblem( + `No version of ${adapterEntry.name} adapter is compatible with your current Gatsby version ${currentGatsbyVersion}.` ) - const required = adaptersRequire.resolve(adapterToUse.module) - - if (required) { - reporter.verbose( - `Reusing existing adapter ${adapterToUse.module} inside .cache/adapters` - ) + } - // TODO: double preferDefault is most ceirtainly wrong - figure it out - return preferDefault(preferDefault(await import(required))) as AdapterInit + { + // 3. Check if the user has manually installed the adapter and try to resolve it from there + const adapterInstalledByUserResults = + await tryLoadingAlreadyInstalledAdapter({ + adapterToUse, + installLocation: process.cwd(), + currentGatsbyVersion, + }) + if (adapterInstalledByUserResults.found) { + if (adapterInstalledByUserResults.compatible) { + reporter.verbose( + `Using site's adapter dependency "${adapterToUse.module}@${adapterInstalledByUserResults.installedVersion}"` + ) + return adapterInstalledByUserResults.loadedModule + } else { + reporter.warn( + `Ignoring incompatible ${adapterToUse.module}@${adapterInstalledByUserResults.installedVersion} installed by site. ${adapterInstalledByUserResults.incompatibilityReason}` + ) + } } - } catch (e) { - // no-op } - const installTimer = reporter.activityTimer( - `Installing ${adapterToUse.name} adapter (${adapterToUse.module}@${versionForCurrentGatsbyVersion.moduleVersion})` - ) - // 4. If both a manually installed version and a cached version are not found, install the adapter into .cache/adapters - try { - installTimer.start() - await createAdaptersCacheDir() + { + // 4. Check if a previous run has installed the correct adapter into .cache/adapters already and try to resolve it from there + const adapterPreviouslyInstalledInCacheAdaptersResults = + await tryLoadingAlreadyInstalledAdapter({ + adapterToUse, + installLocation: getAdaptersCacheDir(), + currentGatsbyVersion, + }) - const options: ExecaOptions = { - stderr: `inherit`, - cwd: getAdaptersCacheDir(), + if (adapterPreviouslyInstalledInCacheAdaptersResults.found) { + if (adapterPreviouslyInstalledInCacheAdaptersResults.compatible) { + reporter.verbose( + `Using previously adapter previously installed by gatsby "${adapterToUse.module}@${adapterPreviouslyInstalledInCacheAdaptersResults.installedVersion}"` + ) + return adapterPreviouslyInstalledInCacheAdaptersResults.loadedModule + } else { + reporter.verbose( + `Ignoring incompatible ${adapterToUse.module} installed by gatsby in ".cache/adapters" before. ${adapterPreviouslyInstalledInCacheAdaptersResults.incompatibilityReason}` + ) + } } + } - const npmAdditionalCliArgs = [ - `--no-progress`, - `--no-audit`, - `--no-fund`, - `--loglevel`, - `error`, - `--color`, - `always`, - `--legacy-peer-deps`, - `--save-exact`, - ] - - await execa( - `npm`, - [ - `install`, - ...npmAdditionalCliArgs, - `${adapterToUse.module}@${versionForCurrentGatsbyVersion.moduleVersion}`, - ], - options + { + // 5. If user has not installed the adapter manually or is incompatible and we don't have cached version installed by gatsby or that version is not compatible + // we try to install compatible version into .cache/adapters + const installTimer = reporter.activityTimer( + `Installing ${adapterToUse.name} adapter (${adapterToUse.module}@${adapterToUse.moduleVersion})` ) - installTimer.end() + try { + installTimer.start() + await createAdaptersCacheDir() - reporter.info( - `If you plan on staying on this deployment platform, consider installing ${adapterToUse.module} as a dependency in your project. This will give you faster and more robust installs.` - ) + const options: ExecaOptions = { + stderr: `inherit`, + cwd: getAdaptersCacheDir(), + } - const adaptersRequire = createRequireFromPath( - `${getAdaptersCacheDir()}/:internal:` - ) - const required = adaptersRequire.resolve(adapterToUse.module) + const npmAdditionalCliArgs = [ + `--no-progress`, + `--no-audit`, + `--no-fund`, + `--loglevel`, + `error`, + `--color`, + `always`, + `--legacy-peer-deps`, + `--save-exact`, + ] - if (required) { - reporter.verbose( - `Using installed adapter ${adapterToUse.module} inside .cache/adapters` + await execa( + `npm`, + [ + `install`, + ...npmAdditionalCliArgs, + `${adapterToUse.module}@${adapterToUse.moduleVersion}`, + ], + options + ) + } catch (e) { + return handleAdapterProblem( + `Could not install adapter "${adapterToUse.module}@${adapterToUse.moduleVersion}". Please install it yourself by adding it to your package.json's dependencies and try building your project again.`, + installTimer.panic ) - - // TODO: double preferDefault is most ceirtainly wrong - figure it out - return preferDefault(preferDefault(await import(required))) as AdapterInit } - } catch (e) { - installTimer.end() - reporter.warn( - `Could not install adapter ${adapterToUse.module}. Please install it yourself by adding it to your package.json's dependencies and try building your project again.` - ) + installTimer.end() } - return undefined + { + // 5. Try to load again from ".cache/adapters" + const adapterAutoInstalledInCacheAdaptersResults = + await tryLoadingAlreadyInstalledAdapter({ + adapterToUse, + installLocation: getAdaptersCacheDir(), + currentGatsbyVersion, + }) + + if (adapterAutoInstalledInCacheAdaptersResults.found) { + if (adapterAutoInstalledInCacheAdaptersResults.compatible) { + reporter.info( + `If you plan on staying on this deployment platform, consider installing "${adapterToUse.module}@${adapterToUse.moduleVersion}" as a dependency in your project. This will give you faster and more robust installs.` + ) + return adapterAutoInstalledInCacheAdaptersResults.loadedModule + } else { + // this indicates a bug as we install version with range from manifest, and now after trying to load the adapter we consider that adapter incompatible + return handleAdapterProblem( + `Auto installed adapter "${adapterToUse.module}@${adapterAutoInstalledInCacheAdaptersResults.installedVersion}"` + ) + } + } else { + // this indicates a bug with adapter itself (fail to resolve main entry point) OR the adapter loading logic + return handleAdapterProblem( + `Could not load adapter "${adapterToUse.module}@${adapterToUse.moduleVersion}". Adapter entry point is not resolvable.` + ) + } + } } diff --git a/packages/gatsby/src/utils/adapter/types.ts b/packages/gatsby/src/utils/adapter/types.ts index c22e15019c5d1..c23dee9667294 100644 --- a/packages/gatsby/src/utils/adapter/types.ts +++ b/packages/gatsby/src/utils/adapter/types.ts @@ -302,5 +302,9 @@ export interface IAdapterManifestEntry { * Version of the adapter. This is a semver range. */ moduleVersion: string + /** + * Can override the module defined in the parent manifest entry - useful for when the adapter is renamed. + */ + module?: string }> } From a263e7b1394d51beaac218b9a6485d1120c10eda Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 22 Dec 2023 18:02:01 +0100 Subject: [PATCH 3/3] test: always allow using installed adapter version in e2e-tests/adapter --- e2e-tests/adapters/gatsby-config.ts | 16 ++++++ .../__tests__/get-latest-gatsby-files.ts | 34 +++++++++++ .../src/utils/adapter/__tests__/init.ts | 56 ++++++++++++++++--- .../src/utils/get-latest-gatsby-files.ts | 29 ++++++---- 4 files changed, 114 insertions(+), 21 deletions(-) diff --git a/e2e-tests/adapters/gatsby-config.ts b/e2e-tests/adapters/gatsby-config.ts index 3bc55e9977d8a..78c0361981ae9 100644 --- a/e2e-tests/adapters/gatsby-config.ts +++ b/e2e-tests/adapters/gatsby-config.ts @@ -15,6 +15,22 @@ if (shouldUseDebugAdapter) { configOverrides = { adapter: debugAdapter(), } +} else { + process.env.GATSBY_ADAPTERS_MANIFEST = /* javascript */ ` + module.exports = [ + { + name: 'Netlify', + module: 'gatsby-adapter-netlify', + test: () => !!process.env.NETLIFY || !!process.env.NETLIFY_LOCAL, + versions: [ + { + gatsbyVersion: '*', + moduleVersion: '*', + } + ], + } + ] + ` } const config: GatsbyConfig = { diff --git a/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts b/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts index 8019bb7bf1606..660d33cd6c72b 100644 --- a/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts +++ b/packages/gatsby/src/utils/__tests__/get-latest-gatsby-files.ts @@ -108,6 +108,9 @@ describe(`default behavior: has network connectivity`, () => { }) describe(`getLatestAdapters`, () => { + beforeEach(() => { + delete process.env.GATSBY_ADAPTERS_MANIFEST + }) it(`loads .js modules (prefers github)`, async () => { axios.get.mockResolvedValueOnce({ data: latestAdaptersMarker }) const data = await getLatestAdapters() @@ -155,6 +158,37 @@ describe(`default behavior: has network connectivity`, () => { expect(data).toEqual(mockAdaptersManifest) }) + + it(`uses GATSBY_ADAPTERS_MANIFEST env var if set`, async () => { + process.env.GATSBY_ADAPTERS_MANIFEST = `custom_manifest` + + axios.get.mockRejectedValueOnce( + new Error(`does not matter and should't be called`) + ) + axios.get.mockRejectedValueOnce( + new Error(`does not matter and should't be called`) + ) + + const data = await getLatestAdapters() + + expect(axios.get).not.toHaveBeenCalledWith( + expect.stringContaining(`raw.githubusercontent.com`), + expect.any(Object) + ) + + expect(axios.get).not.toHaveBeenCalledWith( + expect.stringContaining(`unpkg.com`), + expect.any(Object) + ) + + expect(fs.writeFile).toHaveBeenCalledWith( + expect.stringContaining(`latest-adapters.js`), + process.env.GATSBY_ADAPTERS_MANIFEST, + expect.any(String) + ) + + expect(data).toEqual(mockAdaptersManifest) + }) }) }) diff --git a/packages/gatsby/src/utils/adapter/__tests__/init.ts b/packages/gatsby/src/utils/adapter/__tests__/init.ts index 301d352f5e0cb..9defbff341cf2 100644 --- a/packages/gatsby/src/utils/adapter/__tests__/init.ts +++ b/packages/gatsby/src/utils/adapter/__tests__/init.ts @@ -104,6 +104,18 @@ jest.mock(`gatsby-core-utils/create-require-from-path`, () => { } }) +const getMockedPackageByVersion = (version: string): IMockedAdapterPackage => { + return { + version, + init: (): IAdapter => { + return { + name: `gatsby-adapter-test@${version}`, + adapt: (): void => {}, + } + }, + } +} + const getMockedPackage = ( versionRange: string ): IMockedAdapterPackage | undefined => { @@ -112,15 +124,7 @@ const getMockedPackage = ( versionRange ) if (version) { - return { - version, - init: (): IAdapter => { - return { - name: `gatsby-adapter-test@${version}`, - adapt: (): void => {}, - } - }, - } + return getMockedPackageByVersion(version) } else { return undefined } @@ -538,5 +542,39 @@ describe(`getAdapterInit`, () => { info \\"If you plan on staying on this deployment platform, consider installing /\\"gatsby-adapter-test@^1.0.4/\\" as a dependency in your project. This will give you faster and more robust installs.\\"" `) }) + + it(`gatsby-dev`, async () => { + // gatsby-dev is a special case as it's not published to npm + // it sets package versions to ${current}-dev-${timestamp} and sometimes it's tricky with semver + // as for example 1.0.4-dev-1702672314858 does NOT satisfy ^1.0.4 and normally gatsby would install + // 1.0.4 from npm instead of using version installed in e2e-adapters site via gatsby-dev + // we force specific manifest in e2e-tests/adapters to always allow currently installed adapter version + // via GATSBY_ADAPTERS_MANIFEST env var + mockInstalledInSiteAdapter = getMockedPackageByVersion( + `1.0.4-dev-1702672314858` + ) + mockAdaptersManifest = [ + { + name: `Test`, + test: (): boolean => true, + module: `gatsby-adapter-test`, + versions: [ + { + gatsbyVersion: `*`, + moduleVersion: `*`, + }, + ].filter(Boolean), + }, + ] + + const adapterInit = await getAdapterInit(`5.12.10-dev-1702672314858`) + expect(adapterInit).not.toBeUndefined() + expect(adapterInit?.().name).toMatchInlineSnapshot( + `"gatsby-adapter-test@1.0.4-dev-1702672314858"` + ) + expect(getLogsForSnapshot()).toMatchInlineSnapshot( + `"verbose \\"Using site's adapter dependency /\\"gatsby-adapter-test@1.0.4-dev-1702672314858/\\"\\""` + ) + }) }) }) diff --git a/packages/gatsby/src/utils/get-latest-gatsby-files.ts b/packages/gatsby/src/utils/get-latest-gatsby-files.ts index 6ac5af8c93d9c..20392acab41db 100644 --- a/packages/gatsby/src/utils/get-latest-gatsby-files.ts +++ b/packages/gatsby/src/utils/get-latest-gatsby-files.ts @@ -39,29 +39,32 @@ const _getFile = async ({ fileName, outputFileName, defaultReturn, - tryGithubFirst, + tryGithubBeforeUnpkg, + forcedContent, }: { fileName: string outputFileName: string defaultReturn: T - tryGithubFirst?: boolean + tryGithubBeforeUnpkg?: boolean + forcedContent?: string }): Promise => { let fileToUse = path.join(ROOT, fileName) - let fetchedData = null - if (tryGithubFirst) { - fetchedData = await _fetchFile(GITHUB_ROOT, fileName) + let dataToUse = forcedContent + + if (!dataToUse && tryGithubBeforeUnpkg) { + dataToUse = await _fetchFile(GITHUB_ROOT, fileName) } - if (!fetchedData) { - fetchedData = await _fetchFile(UNPKG_ROOT, fileName) + if (!dataToUse) { + dataToUse = await _fetchFile(UNPKG_ROOT, fileName) } - if (fetchedData) { + if (dataToUse) { await fs.writeFile( outputFileName, - typeof fetchedData === `string` - ? fetchedData - : JSON.stringify(fetchedData, null, 2), + typeof dataToUse === `string` + ? dataToUse + : JSON.stringify(dataToUse, null, 2), `utf8` ) @@ -107,5 +110,7 @@ export const getLatestAdapters = async (): Promise< defaultReturn: [], // trying github first for adapters manifest to be able to faster make changes to version manifest // as publishing latest version of gatsby package takes more time - tryGithubFirst: true, + tryGithubBeforeUnpkg: true, + // in e2e-tests/adapters we force adapters manifest to be used + forcedContent: process.env.GATSBY_ADAPTERS_MANIFEST, })