feat(gatsby-source-drupal): drupal langcode as notlangcode (#37445) (#37459)

* feat(gatsby-source-drupal): drupal langcode as notlangcode (#37445)

* add ts support

* make all files ts

* add new enabled languages plugin option object types/docs

* add local test scripts

* add normalization code for langCode as def

* use proper joi method name

* change property name

* refactor for TS

* ts changes

* ts fixes

* use renamed code in node id

* attempt to disable easlint rules for source-drupal

(cherry picked from commit 3e8bb5a)

* yarn.lock

* fix deps in backport

Co-authored-by: Tyler Barnes <[email protected]>
Co-authored-by: pieh <[email protected]>

Author: 3 people

packages/gatsby-source-drupal/.babelrc

@@ -1,3 +1,3 @@
 {
-  "presets": [["babel-preset-gatsby-package"]]
+  "presets": [["babel-preset-gatsby-package"], "@babel/preset-typescript"]
 }

packages/gatsby-source-drupal/.eslintrc

@@ -0,0 +1,8 @@
+{
+  "rules": {
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/no-use-before-define": "off",
+    "@typescript-eslint/naming-convention": "off",
+    "@typescript-eslint/consistent-type-definitions": "off"
+  }
+}

packages/gatsby-source-drupal/README.md

@@ -465,7 +465,15 @@ module.exports = {
         baseUrl: `https://live-contentacms.pantheonsite.io/`,
         languageConfig: {
           defaultLanguage: `en`,
-          enabledLanguages: [`en`, `fil`],
+          enabledLanguages: [
+            `en`,
+            `fil`,
+            // add an object here if you've renamed a langcode in Drupal
+            {
+              langCode: `en-gb`,
+              as: `uk`,
+            },
+          ],
           translatableEntities: [`node--article`],
           nonTranslatableEntities: [`file--file`],
         },

packages/gatsby-source-drupal/package.json

@@ -24,8 +24,9 @@
     "url-join": "^4.0.1"
   },
   "devDependencies": {
-    "@babel/cli": "^7.15.4",
-    "@babel/core": "^7.15.5",
+    "@babel/cli": "^7.20.7",
+    "@babel/core": "^7.20.7",
+    "@babel/preset-typescript": "^7.18.6",
     "babel-preset-gatsby-package": "^2.25.0",
     "cross-env": "^7.0.3"
   },
@@ -48,8 +49,10 @@
     "directory": "packages/gatsby-source-drupal"
   },
   "scripts": {
-    "build": "babel src --out-dir . --ignore \"**/__tests__\"",
+    "build": "babel src --out-dir . --ignore \"**/__tests__\" --extensions \".ts,.js\"",
     "prepare": "cross-env NODE_ENV=production npm run build",
-    "watch": "babel -w src --out-dir . --ignore \"**/__tests__\""
+    "watch": "babel -w src --out-dir . --ignore \"**/__tests__\" --extensions \".ts,.js\"",
+    "test": "npx jest ./src/__tests__ --runInBand",
+    "test:watch": "npx jest --watch ./src/__tests__ --runInBand"
   }
 }

packages/gatsby-source-drupal/src/__tests__/index.js

@@ -551,7 +551,13 @@ describe(`gatsby-source-drupal`, () => {
         apiBase,
         languageConfig: {
           defaultLanguage: `en_US`,
-          enabledLanguages: [`en_US`, `i18n-test`],
+          enabledLanguages: [
+            `en_US`,
+            {
+              langCode: `en-gb`,
+              as: `i18n-test`,
+            },
+          ],
           translatableEntities: [`node--article`],
           nonTranslatableEntities: [],
         },

packages/gatsby-source-drupal/src/gatsby-node.js renamed to packages/gatsby-source-drupal/src/gatsby-node.ts

@@ -2,6 +2,7 @@ const got = require(`got`)
 const _ = require(`lodash`)
 const urlJoin = require(`url-join`)
 import HttpAgent from "agentkeepalive"
+
 // const http2wrapper = require(`http2-wrapper`)
 const opentracing = require(`opentracing`)
 const { SemanticAttributes } = require(`@opentelemetry/semantic-conventions`)
@@ -15,12 +16,13 @@ const { HttpsAgent } = HttpAgent
 
 const { setOptions, getOptions } = require(`./plugin-options`)
 
-const {
+import {
   nodeFromData,
   downloadFile,
   isFileNode,
   imageCDNState,
-} = require(`./normalize`)
+} from "./normalize"
+
 const {
   handleReferences,
   handleWebhookUpdate,
@@ -567,15 +569,19 @@ ${JSON.stringify(webhookBody, null, 4)}`
               throw error
             }
           }
+
           if (d.body.data) {
-            dataArray.push(...d.body.data)
+            // @ts-ignore
+            dataArray.push(...(d.body.data || []))
           }
+
           // Add support for includes. Includes allow entity data to be expanded
           // based on relationships. The expanded data is exposed as `included`
           // in the JSON API response.
           // See https://www.drupal.org/docs/8/modules/jsonapi/includes
           if (d.body.included) {
-            dataArray.push(...d.body.included)
+            // @ts-ignore
+            dataArray.push(...(d.body.included || []))
           }
 
           // If JSON:API extras is configured to add the resource count, we can queue
@@ -593,8 +599,8 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
               // Get count of API requests
               // We round down as we've already gotten the first page at this point.
-              const pageSize = new URL(d.body.links.next.href).searchParams.get(
-                `page[limit]`
+              const pageSize = Number(
+                new URL(d.body.links.next.href).searchParams.get(`page[limit]`)
               )
               const requestsCount = Math.floor(d.body.meta.count / pageSize)
 
@@ -604,11 +610,14 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
               const newUrl = new URL(d.body.links.next.href)
               await Promise.all(
-                _.range(requestsCount).map(pageOffset => {
+                _.range(requestsCount).map((pageOffset: number) => {
                   // We're starting 1 ahead.
                   pageOffset += 1
                   // Construct URL with new pageOffset.
-                  newUrl.searchParams.set(`page[offset]`, pageOffset * pageSize)
+                  newUrl.searchParams.set(
+                    `page[offset]`,
+                    String(pageOffset * pageSize)
+                  )
                   return getNext(newUrl.toString(), currentLanguage)
                 })
               )
@@ -626,6 +635,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
             const urlPath = url.href.split(`${apiBase}/`).pop()
             const baseUrlWithoutTrailingSlash = baseUrl.replace(/\/$/, ``)
             // The default language's JSON API is at the root.
+
             if (
               currentLanguage === getOptions().languageConfig.defaultLanguage ||
               baseUrlWithoutTrailingSlash.slice(-currentLanguage.length) ==
@@ -869,7 +879,15 @@ exports.pluginOptionsSchema = ({ Joi }) =>
     ),
     languageConfig: Joi.object({
       defaultLanguage: Joi.string().required(),
-      enabledLanguages: Joi.array().items(Joi.string()).required(),
+      enabledLanguages: Joi.array()
+        .items(
+          Joi.string(),
+          Joi.object({
+            langCode: Joi.string().required(),
+            as: Joi.string().required(),
+          })
+        )
+        .required(),
       translatableEntities: Joi.array().items(Joi.string()).required(),
       nonTranslatableEntities: Joi.array().items(Joi.string()).required(),
     }),

packages/gatsby-source-drupal/src/normalize.js renamed to packages/gatsby-source-drupal/src/normalize.ts

@@ -3,23 +3,20 @@ const { createRemoteFileNode } = require(`gatsby-source-filesystem`)
 const path = require(`path`)
 const probeImageSize = require(`probe-image-size`)
 
-const { getOptions } = require(`./plugin-options`)
-const getHref = link => {
+import { getOptions } from "./plugin-options"
+
+export const getHref = link => {
   if (typeof link === `object`) {
     return link.href
   }
   return link
 }
 
-exports.getHref = getHref
-
-const imageCDNState = {
+export const imageCDNState = {
   foundPlaceholderStyle: false,
   hasLoggedNoPlaceholderStyle: false,
 }
 
-exports.imageCDNState = imageCDNState
-
 let four04WarningCount = 0
 let corruptFileWarningCount = 0
 /**
@@ -142,17 +139,20 @@ const getGatsbyImageCdnFields = async ({
   return {}
 }
 
-const nodeFromData = async (
+export const nodeFromData = async (
   datum,
   createNodeId,
   entityReferenceRevisions = [],
   pluginOptions,
   fileNodesExtendedData,
   reporter
 ) => {
-  const { attributes: { id: attributeId, ...attributes } = {} } = datum
+  const { attributes: { id: attributeId, ...attributes } = { id: null } } =
+    datum
+
   const preservedId =
     typeof attributeId !== `undefined` ? { _attributes_id: attributeId } : {}
+
   const langcode = attributes.langcode || `und`
   const type = datum.type.replace(/-|__|:|\.|\s/g, `_`)
 
@@ -164,16 +164,18 @@ const nodeFromData = async (
     reporter,
   })
 
+  const versionedId = createNodeIdWithVersion(
+    datum.id,
+    datum.type,
+    langcode,
+    attributes.drupal_internal__revision_id,
+    entityReferenceRevisions
+  )
+
+  const gatsbyId = createNodeId(versionedId)
+
   return {
-    id: createNodeId(
-      createNodeIdWithVersion(
-        datum.id,
-        datum.type,
-        langcode,
-        attributes.drupal_internal__revision_id,
-        entityReferenceRevisions
-      )
-    ),
+    id: gatsbyId,
     drupal_id: datum.id,
     parent: null,
     drupal_parent_menu_item: attributes.parent,
@@ -189,52 +191,68 @@ const nodeFromData = async (
   }
 }
 
-exports.nodeFromData = nodeFromData
-
 const isEntityReferenceRevision = (type, entityReferenceRevisions = []) =>
   entityReferenceRevisions.findIndex(
     revisionType => type.indexOf(revisionType) === 0
   ) !== -1
 
-const createNodeIdWithVersion = (
-  id,
-  type,
-  langcode,
-  revisionId,
+export const createNodeIdWithVersion = (
+  id: string,
+  type: string,
+  langcode: string,
+  revisionId: string,
   entityReferenceRevisions = []
 ) => {
+  const options = getOptions()
+
   // Fallback to default language for entities that don't translate.
-  if (getOptions()?.languageConfig?.nonTranslatableEntities.includes(type)) {
-    langcode = getOptions().languageConfig.defaultLanguage
+  if (
+    options?.languageConfig?.nonTranslatableEntities?.includes(type) &&
+    options.languageConfig.defaultLanguage
+  ) {
+    langcode = options.languageConfig.defaultLanguage
   }
 
   // If the source plugin hasn't enabled `translation` then always just set langcode
   // to "undefined".
-  let langcodeNormalized = getOptions().languageConfig ? langcode : `und`
+  let langcodeNormalized = options.languageConfig ? langcode : `und`
+
+  const renamedCode = options?.languageConfig?.renamedEnabledLanguages?.find(
+    lang => lang.langCode === langcodeNormalized
+  )
+
+  if (renamedCode) {
+    langcodeNormalized = renamedCode.as
+  }
 
   if (
-    getOptions().languageConfig &&
-    !getOptions().languageConfig?.enabledLanguages.includes(langcodeNormalized)
+    !renamedCode &&
+    options.languageConfig &&
+    options.languageConfig.defaultLanguage &&
+    !options?.languageConfig?.enabledLanguages?.includes(langcodeNormalized)
   ) {
-    langcodeNormalized = getOptions().languageConfig.defaultLanguage
+    langcodeNormalized = options.languageConfig.defaultLanguage
   }
 
+  const isReferenceRevision = isEntityReferenceRevision(
+    type,
+    entityReferenceRevisions
+  )
+
   // The relationship between an entity and another entity also depends on the revision ID if the field is of type
   // entity reference revision such as for paragraphs.
-  return isEntityReferenceRevision(type, entityReferenceRevisions)
+  const idVersion = isReferenceRevision
     ? `${langcodeNormalized}.${id}.${revisionId || 0}`
     : `${langcodeNormalized}.${id}`
-}
 
-exports.createNodeIdWithVersion = createNodeIdWithVersion
+  return idVersion
+}
 
-const isFileNode = node => {
+export const isFileNode = node => {
   const type = node?.internal?.type
   return type === `files` || type === `file__file`
 }
 
-exports.isFileNode = isFileNode
-
 const getFileUrl = (node, baseUrl) => {
   let fileUrl = node.url
 
@@ -249,8 +267,8 @@ const getFileUrl = (node, baseUrl) => {
   return url
 }
 
-exports.downloadFile = async (
-  { node, store, cache, createNode, createNodeId, getCache, reporter },
+export const downloadFile = async (
+  { node, cache, createNode, createNodeId, getCache },
   { basicAuth, baseUrl }
 ) => {
   // handle file downloads