Merge pull request #594 from newrelic/jerel/sync-search-data

Source related resource content from Swiftype

gatsby-config.js

@@ -1,3 +1,5 @@
+const quote = (str) => `"${str}"`;
+
 module.exports = {
   siteMetadata: {
     title: 'New Relic Developers',
@@ -42,6 +44,51 @@ module.exports = {
         },
       },
     },
+    {
+      resolve: 'gatsby-source-swiftype',
+      options: {
+        file: `${__dirname}/src/data/related-pages.json`,
+        refetch: Boolean(process.env.BUILD_RELATED_CONTENT),
+        engineKey: 'Ad9HfGjDw4GRkcmJjUut',
+        limit: 5,
+        getPath: ({ node }) => node.frontmatter.path,
+        getParams: ({ node }) => {
+          const {
+            tags,
+            title,
+            redirects = [],
+            resources = [],
+          } = node.frontmatter;
+
+          const filteredUrls = resources
+            .map((resource) => resource.url)
+            .concat(redirects);
+
+          return {
+            q: tags ? tags.map(quote).join(' OR ') : title,
+            search_fields: {
+              page: ['tags^10', 'body^5', 'title^1.5', '*'],
+            },
+            filters: {
+              page: {
+                type: ['!blog', '!forum'],
+                url: filteredUrls.map((url) =>
+                  url.startsWith('/')
+                    ? `!https://developer.newrelic.com${url}`
+                    : `!${url}`
+                ),
+                document_type: [
+                  '!views_page_menu',
+                  '!term_page_api_menu',
+                  '!term_page_landing_page',
+                ],
+              },
+            },
+          };
+        },
+        filterNode: ({ node }) => node.frontmatter.template === 'GuideTemplate',
+      },
+    },
     'gatsby-plugin-sass',
     {
       resolve: 'gatsby-plugin-manifest',

package.json

@@ -31,6 +31,7 @@
     "gatsby-transformer-remark": "^2.8.23",
     "gatsby-transformer-sharp": "^2.5.10",
     "js-cookie": "^2.2.1",
+    "node-fetch": "^2.6.0",
     "node-sass": "^4.14.1",
     "prism-react-renderer": "^1.1.1",
     "prismjs": "^1.21.0",
@@ -71,6 +72,7 @@
     "build": "gatsby build",
     "build:production": "GATSBY_NEWRELIC_ENV=production gatsby build",
     "build:staging": "GATSBY_NEWRELIC_ENV=staging gatsby build",
+    "build:related-content": "BUILD_RELATED_CONTENT=true npm run build:production",
     "develop": "gatsby develop",
     "format": "prettier --write \"**/*.{js,jsx,json,md}\"",
     "start": "npm run develop",

plugins/gatsby-source-swiftype/gatsby-node.js

@@ -0,0 +1,93 @@
+const fs = require('fs');
+const createRelatedResourceNode = require('./src/createRelatedResourceNode');
+const getRelatedResources = require('./src/getRelatedResources');
+
+const writeableData = {};
+
+exports.onPreBootstrap = (_, pluginOptions) => {
+  const { file } = pluginOptions;
+
+  if (!fs.existsSync(file)) {
+    fs.writeFileSync(file, '{}');
+  }
+};
+
+exports.onCreateNode = async (
+  { actions, node, getNodesByType, createNodeId, createContentDigest },
+  pluginOptions
+) => {
+  const { createNode, createParentChildLink } = actions;
+  const { filterNode = () => false, getPath } = pluginOptions;
+
+  if (node.internal.type !== 'Mdx' || !filterNode({ node })) {
+    return;
+  }
+
+  const [
+    {
+      siteMetadata: { siteUrl },
+    },
+  ] = getNodesByType('Site');
+
+  const pathname = getPath({ node });
+  const resources = await getRelatedResources({ node, siteUrl }, pluginOptions);
+
+  writeableData[pathname] = resources;
+
+  resources.forEach((resource) => {
+    const child = createRelatedResourceNode({
+      parent: node.id,
+      resource,
+      createContentDigest,
+      createNode,
+      createNodeId,
+    });
+
+    createParentChildLink({ parent: node, child: child });
+  });
+};
+
+exports.createSchemaCustomization = ({ actions }) => {
+  const { createTypes } = actions;
+
+  const typeDefs = `
+  type RelatedResource implements Node {
+    id: ID!
+    title: String!
+    url: String!
+  }
+  `;
+
+  createTypes(typeDefs);
+};
+
+exports.createResolvers = ({ createResolvers }) => {
+  createResolvers({
+    Mdx: {
+      relatedResources: {
+        args: {
+          limit: {
+            type: 'Int',
+            defaultValue: 5,
+          },
+        },
+        type: ['RelatedResource!'],
+        resolve: (source, args, context) => {
+          const { limit } = args;
+
+          return context.nodeModel
+            .getNodesByIds({ ids: source.children })
+            .slice(0, Math.max(limit, 0));
+        },
+      },
+    },
+  });
+};
+
+exports.onPostBootstrap = (_, pluginOptions) => {
+  const { refetch, file } = pluginOptions;
+
+  if (refetch) {
+    fs.writeFileSync(file, JSON.stringify(writeableData, null, 2));
+  }
+};

plugins/gatsby-source-swiftype/package.json

@@ -0,0 +1 @@
+{}

plugins/gatsby-source-swiftype/src/createRelatedResourceNode.js

@@ -0,0 +1,25 @@
+module.exports = ({
+  createNode,
+  createNodeId,
+  createContentDigest,
+  resource,
+  parent,
+}) => {
+  const node = {
+    id: createNodeId(`RelatedResource-${resource.url}`),
+    title: resource.title,
+    url: resource.url,
+    parent,
+    children: [],
+    plugin: 'gatsby-source-swiftype',
+    internal: {
+      type: 'RelatedResource',
+      content: JSON.stringify(resource),
+      contentDigest: createContentDigest(resource),
+    },
+  };
+
+  createNode(node);
+
+  return node;
+};

plugins/gatsby-source-swiftype/src/getRelatedResources.js

@@ -0,0 +1,26 @@
+const fs = require('fs');
+const search = require('./search');
+
+module.exports = async ({ node, siteUrl }, pluginOptions) => {
+  const {
+    refetch,
+    engineKey,
+    limit,
+    file,
+    getParams = () => ({}),
+    getPath,
+  } = pluginOptions;
+
+  const pathname = getPath({ node });
+
+  if (refetch) {
+    return search(siteUrl + pathname, getParams({ node }), {
+      engineKey,
+      limit,
+    });
+  }
+
+  const data = JSON.parse(fs.readFileSync(file));
+
+  return data[pathname] || [];
+};

plugins/gatsby-source-swiftype/src/search.js

@@ -0,0 +1,47 @@
+const fetch = require('node-fetch');
+const { appendTrailingSlash, stripTrailingSlash } = require('./utils/url');
+
+const normalizeUrl = (url) => {
+  const prefix = url.startsWith('!') ? '!' : '';
+  const plainUrl = url.replace(/^!/, '');
+
+  return [
+    prefix + appendTrailingSlash(plainUrl),
+    prefix + stripTrailingSlash(plainUrl),
+  ];
+};
+
+const uniq = (arr) => [...new Set(arr)];
+
+module.exports = async (url, params = {}, { engineKey, limit }) => {
+  const { page: pageFilters = {} } = params.filters || {};
+
+  const res = await fetch(
+    'https://search-api.swiftype.com/api/v1/public/engines/search.json',
+    {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        ...params,
+        engine_key: engineKey,
+        per_page: limit,
+        filters: {
+          ...params.filters,
+          page: {
+            ...pageFilters,
+            url: uniq([
+              ...normalizeUrl(`!${url}`),
+              ...(pageFilters.url || []).flatMap(normalizeUrl),
+            ]),
+          },
+        },
+      }),
+    }
+  );
+
+  const { records } = await res.json();
+
+  return records.page;
+};

plugins/gatsby-source-swiftype/src/utils/url.js

@@ -0,0 +1,21 @@
+const hasQueryParams = (urlString) => {
+  const url = new URL(urlString);
+
+  return Boolean(url.search);
+};
+
+exports.appendTrailingSlash = (url) => {
+  if (hasQueryParams(url)) {
+    return url;
+  }
+
+  return url.endsWith('/') ? url : `${url}/`;
+};
+
+exports.stripTrailingSlash = (url) => {
+  if (hasQueryParams(url)) {
+    return url;
+  }
+
+  return url.endsWith('/') ? url.replace(/\/$/, '') : url;
+};