Replace data/*.json files with build time .cache/site.json (#7074)

**This PR**
- [x] Adds a build time script to generate the `.cache/site.json` file making sure the navigation and search do not need to be updated when creating new pages
- [x] Replaces `components.json` and `foundations.json` with `.cache/site.json`
- [x] Uses one file for the site data so that there is one `readFile` script needed to access it
- [x] Uses one file for the site data so it can be cached effectively when read
- [x] **Local dev:** Make the `.cache/site.json` file regenerate on changes to markdown files
- [x] Fixes json file typescript issues

**Future work**
- [ ] **Production:** Do not read and store `.cache/site.json` in client side memory when generating the navigation
- [ ] **Production:** Do not read and store `.cache/site.json` in client side memory when generating the components page categories

Co-authored-by: Chloe Rice <[email protected]>

Author: alex-page

.changeset/eighty-cups-retire.md

@@ -0,0 +1,5 @@
+---
+'polaris.shopify.com': minor
+---
+
+Replace data/\*.json files with build time .cache/site.json

polaris.shopify.com/package.json

@@ -4,8 +4,10 @@
   "private": true,
   "scripts": {
     "build": "yarn gen-assets && next build",
-    "dev": "open http://localhost:3000 && next dev",
     "start": "next start",
+    "dev": "run-p dev:*",
+    "dev:server": "open http://localhost:3000 && next dev",
+    "dev:watch-md": "node scripts/watch-md.mjs",
     "lint": "run-p lint:*",
     "lint:js": "TIMING=1 eslint --cache .",
     "lint:styles": "stylelint '**/*.{css,scss}'",
@@ -40,6 +42,7 @@
     "@types/prismjs": "^1.26.0",
     "@types/react": "*",
     "@types/lodash.throttle": "^4.1.7",
+    "chokidar": "^3.5.3",
     "eslint-config-next": "12.1.0",
     "eslint": "8.10.0",
     "execa": "^6.1.0",

polaris.shopify.com/pages/api/search/v0/index.tsx

@@ -1,20 +1,29 @@
 import type {NextApiRequest, NextApiResponse} from 'next';
 import Fuse from 'fuse.js';
+import {metadata, MetadataProperties} from '@shopify/polaris-tokens';
+import iconMetadata from '@shopify/polaris-icons/metadata';
 
 import {
   SearchResults,
   GroupedSearchResults,
   searchResultCategories,
   SearchResultCategory,
   Status,
+  SiteJSON,
 } from '../../../../src/types';
 
 import {slugify, stripMarkdownLinks} from '../../../../src/utils/various';
 
-import {metadata, MetadataProperties} from '@shopify/polaris-tokens';
-import iconMetadata from '@shopify/polaris-icons/metadata';
-import components from '../../../../src/data/components.json';
-import foundations from '../../../../src/data/foundations.json';
+import siteJson from '../../../../.cache/site.json';
+
+const pages: SiteJSON = siteJson;
+
+const componentSlugs = Object.keys(pages).filter((slug) =>
+  slug.startsWith('components/'),
+);
+const foundationSlugs = Object.keys(pages).filter((slug) =>
+  slug.startsWith('foundations/'),
+);
 
 const MAX_RESULTS: {[key in SearchResultCategory]: number} = {
   foundations: 8,
@@ -29,13 +38,8 @@ const getSearchResults = (query: string) => {
   let results: SearchResults = [];
 
   // Add components
-  components.forEach(({frontMatter: {title, status}, description}) => {
-    const typedStatus: Status | undefined = status
-      ? {
-          value: status.value.toLowerCase() as Status['value'],
-          message: status.message,
-        }
-      : undefined;
+  componentSlugs.forEach((slug) => {
+    const {status, title, description = ''} = pages[slug].frontMatter;
 
     results.push({
       id: slugify(`components ${title}`),
@@ -46,7 +50,7 @@ const getSearchResults = (query: string) => {
         components: {
           title,
           description: stripMarkdownLinks(description),
-          status: typedStatus,
+          status: status as Status,
         },
       },
     });
@@ -102,20 +106,19 @@ const getSearchResults = (query: string) => {
   });
 
   // Add foundations
-  foundations.forEach((data) => {
-    const {title, icon} = data.frontMatter;
-    const {description, category} = data;
-    const url = `/foundations/${category}/${slugify(title)}`;
+  foundationSlugs.forEach((slug) => {
+    const {title, icon = '', description = ''} = pages[slug].frontMatter;
+    const category = slug.split('/')[2];
 
     results.push({
       id: slugify(`foundations ${title}`),
       category: 'foundations',
       score: 0,
-      url,
+      url: slug,
       meta: {
         foundations: {
           title,
-          icon: icon || '',
+          icon,
           description,
           category: category || '',
         },

polaris.shopify.com/pages/components/[component].tsx

@@ -42,22 +42,20 @@ const Components = ({
   propsForComponent,
 }: Props) => {
   const navItems: NavItem[] = getComponentNav();
-  const typedStatus: Status | undefined = status
-    ? {
-        value: status.value.toLowerCase() as Status['value'],
-        message: status.message,
-      }
-    : undefined;
+  const statusBanner = status ? <StatusBanner status={status} /> : null;
+  const propList = propsForComponent ? (
+    <PropsTable props={propsForComponent} />
+  ) : null;
 
   return (
     <Layout width="narrow" navItems={navItems} title={title}>
       <PageMeta title={title} description={description} />
 
       <Longform>
         <Markdown text={description} />
-        {typedStatus && <StatusBanner status={typedStatus} />}
+        {statusBanner}
         <ComponentExamples examples={examples} />
-        {propsForComponent && <PropsTable props={propsForComponent} />}
+        {propList}
         <Markdown text={readme.body} />
       </Longform>
     </Layout>

polaris.shopify.com/scripts/gen-site-json.mjs

@@ -1,14 +1,13 @@
 import path from 'path';
 import globby from 'globby';
-import {rm, mkdir, writeFile, readFile} from 'fs/promises';
-import {existsSync} from 'fs';
+import {existsSync, rmSync, mkdirSync, writeFileSync, readFileSync} from 'fs';
 import matter from 'gray-matter';
 
 const cacheDir = path.join(process.cwd(), '.cache');
 const siteJsonFile = `${cacheDir}/site.json`;
 
-const getMdContent = async (filePath) => {
-  const fileContent = await readFile(filePath, 'utf-8');
+const getMdContent = (filePath) => {
+  const fileContent = readFileSync(filePath, 'utf-8');
   const {data, content} = matter(fileContent);
   const slug = filePath
     .replace(`${process.cwd()}/content/`, '')
@@ -17,23 +16,19 @@ const getMdContent = async (filePath) => {
   return {frontMatter: data, slug};
 };
 
-const genSiteJson = async () => {
-  if (existsSync(cacheDir)) await rm(cacheDir, {recursive: true});
-  await mkdir(cacheDir, {recursive: true});
+const genSiteJson = () => {
+  if (!existsSync(cacheDir)) mkdirSync(cacheDir, {recursive: true});
   const pathGlob = path.join(process.cwd(), 'content/**/*.md');
   const mdFiles = globby.sync(pathGlob);
 
-  const mdDataPromise = mdFiles.map((filePath) => getMdContent(filePath));
-  const mdData = await Promise.all(mdDataPromise);
+  const mdData = mdFiles.map((filePath) => getMdContent(filePath));
 
   const data = {};
   mdData.forEach((md) => (data[md.slug] = {frontMatter: md.frontMatter}));
 
-  await writeFile(siteJsonFile, JSON.stringify(data), 'utf-8');
+  writeFileSync(siteJsonFile, JSON.stringify(data), 'utf-8');
 
   console.log(`✅ Generated ${siteJsonFile}`);
 };
 
-genSiteJson();
-
 export default genSiteJson;

polaris.shopify.com/scripts/watch-md.mjs

@@ -0,0 +1,8 @@
+import path from 'path';
+import chokidar from 'chokidar';
+
+import genSiteJson from './gen-site-json.mjs';
+
+const mdPath = path.join(process.cwd(), 'content');
+
+chokidar.watch(mdPath).on('change', async () => await genSiteJson());

polaris.shopify.com/src/components/ComponentGrid/ComponentGrid.tsx

@@ -1,6 +1,6 @@
 import Image from '../Image';
 import Link from 'next/link';
-import {getReadableStatusValue, slugify} from '../../utils/various';
+import {slugify} from '../../utils/various';
 import {Status} from '../../types';
 import styles from './ComponentGrid.module.scss';
 import StatusBadge from '../StatusBadge';
@@ -28,6 +28,9 @@ function ComponentGridItem({
   status,
 }: ComponentGridItemProps) {
   const searchAttributes = useGlobalSearchResult();
+  const statusBadge = status ? (
+    <StatusBadge status={{value: status.value, message: status.value}} />
+  ) : null;
 
   return (
     <li key={title} className={styles.Component} {...searchAttributes}>
@@ -47,18 +50,7 @@ function ComponentGridItem({
           </div>
           <div className={styles.ComponentDescription}>
             <h4>
-              {title}
-              {status && (
-                <>
-                  {' '}
-                  <StatusBadge
-                    status={{
-                      value: status.value,
-                      message: getReadableStatusValue(status.value),
-                    }}
-                  />
-                </>
-              )}
+              {`${title} `} {statusBadge}
             </h4>
             <p>{description}</p>
           </div>

polaris.shopify.com/src/components/ComponentsPage/ComponentsPage.tsx

@@ -1,17 +1,23 @@
 import ComponentGrid from '../ComponentGrid';
 import Layout from '../Layout';
 
-import components from '../../data/components.json';
+import siteJson from '../../../.cache/site.json';
 import {
   getComponentCategories,
   stripMarkdownLinks,
   slugify,
   getComponentNav,
 } from '../../utils/various';
-import {Status} from '../../types';
+import {Status, SiteJSON} from '../../types';
 import styles from './ComponentsPage.module.scss';
 import PageMeta from '../PageMeta';
 
+const pages: SiteJSON = siteJson;
+
+const components = Object.keys(pages).filter((slug) =>
+  slug.startsWith('components/'),
+);
+
 const componentCategories = getComponentCategories();
 const componentNav = getComponentNav();
 
@@ -33,25 +39,22 @@ export default function ComponentsPage() {
               <ComponentGrid>
                 {components
                   .filter(
-                    (component) => component.frontMatter.category === category,
+                    (slug) => pages[slug].frontMatter.category === category,
                   )
-                  .map(({frontMatter, description}) => {
-                    const {title, status} = frontMatter;
+                  .map((slug) => {
+                    const {
+                      title,
+                      status,
+                      description = '',
+                    } = pages[slug].frontMatter;
                     const url = `/components/${slugify(title)}`;
-                    let typedStatus = status
-                      ? {
-                          value: status.value.toLowerCase() as Status['value'],
-                          message: status.value,
-                        }
-                      : undefined;
-
                     return (
                       <ComponentGrid.Item
                         key={title}
                         title={title}
                         description={stripMarkdownLinks(description)}
                         url={url}
-                        status={typedStatus}
+                        status={status as Status}
                       />
                     );
                   })}

polaris.shopify.com/src/components/PropsTable/PropsTable.tsx

@@ -1,4 +1,4 @@
-import {PropsForComponent} from '../../types';
+import {PropsForComponent, StatusName} from '../../types';
 import StatusBadge from '../StatusBadge';
 import styles from './PropsTable.module.scss';
 
@@ -36,7 +36,7 @@ function PropsTable({props: {props}}: Props) {
                       {' '}
                       <StatusBadge
                         status={{
-                          value: 'information',
+                          value: StatusName.Information,
                           message: 'Required',
                         }}
                       />
@@ -47,7 +47,7 @@ function PropsTable({props: {props}}: Props) {
                       {' '}
                       <StatusBadge
                         status={{
-                          value: 'deprecated',
+                          value: StatusName.Deprecated,
                           message: 'Deprecated',
                         }}
                       />

polaris.shopify.com/src/components/StatusBadge/StatusBadge.tsx

@@ -4,10 +4,10 @@ import styles from './StatusBadge.module.scss';
 interface Props {
   status: Status;
 }
-function StatusBadge({status}: Props) {
+function StatusBadge({status: {value, message}}: Props) {
   return (
-    <div className={styles.StatusBadge} data-value={status.value}>
-      {status.message}
+    <div className={styles.StatusBadge} data-value={value.toLowerCase()}>
+      {message}
     </div>
   );
 }