feature: scaffold new fern-adapter package to be run as cron

Author: skeptrunedev

clients/trieve-fern-adapter/index.ts

@@ -1,11 +1,15 @@
+#!/usr/bin/env node
+
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { Command } from 'commander';
 import { parse } from 'yaml';
 import { Window } from 'happy-dom';
 import fs from 'node:fs';
+import { join } from 'node:path';
 import { TrieveSDK, ChunkReqPayload } from 'trieve-ts-sdk';
 import { marked } from 'marked';
 import { dereferenceSync } from '@trojs/openapi-dereference';
+import pluralize from 'pluralize'
 
 const splitHtmlIntoHeadAndBodies = (html: Element): [string, string][] => {
   const headingRegex = /h\d/gi;
@@ -120,7 +124,7 @@ const extractChunksFromPath = async (
       .filter((x) => x);
     const metadata: any = {
       url: link,
-      heirarchy: tag_set,
+      hierarchy: tag_set,
       heading: heading,
     };
 
@@ -196,11 +200,13 @@ const extractChunksFromOpenapiSpec = async (
         const operationId = pathData[method].operationId;
         const summary = pathData[method].summary;
         const description = pathData[method].description;
-        const pageLink = `${siteUrl}/${apiRefParent}/${summary?.split(' ').join('-').toLowerCase() ?? path}`;
+        const [namespace, ...parts] = summary?.toLowerCase().split(' ') ?? []
+        const endpoint = namespace ? join(pluralize(parts.join('-')), namespace) : path
+        const pageLink = `${siteUrl}/${apiRefParent}/${endpoint}`;
         const metadata = {
           operation_id: operationId,
           url: pageLink,
-          heirarchy: [
+          hierarchy: [
             apiRefParent,
             summary?.split(' ').join('-').toLowerCase() ?? path,
           ],
@@ -337,8 +343,8 @@ try {
     }
     console.info('Waiting on delete...');
   }
-} catch (err) {
-  console.info('Dataset not found, creating...', err);
+} catch {
+  console.info('Dataset not found, creating...');
   try {
     const createdDataset = await trieve.createDataset({
       tracking_id: trieveDatasetTrackingId,
@@ -366,4 +372,4 @@ for (let i = 0; i < chunkReqPayloads.length; i += 120) {
 }
 
 console.log('Done!');
-process.exit(0);
+process.exit(0);

clients/trieve-fern-adapter/package.json

@@ -15,6 +15,7 @@
   "devDependencies": {
     "@eslint/js": "^9.15.0",
     "@types/node": "^22.10.0",
+    "@types/pluralize": "^0.0.33",
     "@typescript-eslint/eslint-plugin": "^8.16.0",
     "@typescript-eslint/parser": "^8.16.0",
     "eslint": "^9.15.0",
@@ -29,6 +30,7 @@
     "commander": "^12.1.0",
     "happy-dom": "^15.11.6",
     "marked": "^15.0.2",
+    "pluralize": "^8.0.0",
     "trieve-ts-sdk": "^0.0.35",
     "yaml": "^2.6.1"
   }

clients/trieve-fern-adapter/yarn.lock

@@ -45,7 +45,7 @@
 
 "@eslint/[email protected]", "@eslint/js@^9.15.0":
   version "9.15.0"
-  resolved "https://registry.npmjs.org/@eslint/js/-/js-9.15.0.tgz"
+  resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.15.0.tgz#df0e24fe869143b59731942128c19938fdbadfb5"
   integrity sha512-tMTqrY+EzbXmKJR5ToI8lxu7jaN5EdmrBFJpQk5JmSlyLsx6o4t27r883K5xsLuCYCpfKBCGswMSWXsM+jB7lg==
 
 "@eslint/object-schema@^2.1.4":
@@ -131,6 +131,11 @@
   dependencies:
     undici-types "~6.20.0"
 
+"@types/pluralize@^0.0.33":
+  version "0.0.33"
+  resolved "https://registry.yarnpkg.com/@types/pluralize/-/pluralize-0.0.33.tgz#8ad9018368c584d268667dd9acd5b3b806e8c82a"
+  integrity sha512-JOqsl+ZoCpP4e8TDke9W79FDcSgPAR0l6pixx2JHkhnRjvShyYiAYw2LVsnA7K08Y6DeOnaU6ujmENO4os/cYg==
+
 "@typescript-eslint/[email protected]", "@typescript-eslint/eslint-plugin@^8.16.0":
   version "8.16.0"
   resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.16.0.tgz"
@@ -298,7 +303,7 @@ color-name@~1.1.4:
 
 commander@^12.1.0:
   version "12.1.0"
-  resolved "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-12.1.0.tgz#01423b36f501259fdaac4d0e4d60c96c991585d3"
   integrity sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==
 
 [email protected]:
@@ -329,7 +334,7 @@ deep-is@^0.1.3:
 
 entities@^4.5.0:
   version "4.5.0"
-  resolved "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48"
   integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==
 
 escape-string-regexp@^4.0.0:
@@ -339,7 +344,7 @@ escape-string-regexp@^4.0.0:
 
 eslint-config-prettier@^9.1.0:
   version "9.1.0"
-  resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz"
+  resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz#31af3d94578645966c082fcb71a5846d3c94867f"
   integrity sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==
 
 eslint-scope@^8.2.0:
@@ -362,7 +367,7 @@ eslint-visitor-keys@^4.2.0:
 
 eslint@^9.15.0:
   version "9.15.0"
-  resolved "https://registry.npmjs.org/eslint/-/eslint-9.15.0.tgz"
+  resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.15.0.tgz#77c684a4e980e82135ebff8ee8f0a9106ce6b8a6"
   integrity sha512-7CrWySmIibCgT1Os28lUU6upBshZ+GxybLOrmRzi08kS8MBuO8QA7pXEgYgY5W8vK3e74xv0lpjo9DbaGU9Rkw==
   dependencies:
     "@eslint-community/eslint-utils" "^4.2.0"
@@ -522,7 +527,7 @@ globals@^14.0.0:
 
 globals@^15.12.0:
   version "15.12.0"
-  resolved "https://registry.npmjs.org/globals/-/globals-15.12.0.tgz"
+  resolved "https://registry.yarnpkg.com/globals/-/globals-15.12.0.tgz#1811872883ad8f41055b61457a130221297de5b5"
   integrity sha512-1+gLErljJFhbOVyaetcwJiJ4+eLe45S2E7P5UiZ9xGfeq3ATQf5DOv9G7MH3gGbKQLkzmNh2DxfZwLdw+j6oTQ==
 
 graphemer@^1.4.0:
@@ -532,7 +537,7 @@ graphemer@^1.4.0:
 
 happy-dom@^15.11.6:
   version "15.11.6"
-  resolved "https://registry.npmjs.org/happy-dom/-/happy-dom-15.11.6.tgz"
+  resolved "https://registry.yarnpkg.com/happy-dom/-/happy-dom-15.11.6.tgz#b40d46060114fee5c4f0e79c6a1aec50cd42e4c8"
   integrity sha512-elX7iUTu+5+3b2+NGQc0L3eWyq9jKhuJJ4GpOMxxT/c2pg9O3L5H3ty2VECX0XXZgRmmRqXyOK8brA2hDI6LsQ==
   dependencies:
     entities "^4.5.0"
@@ -586,7 +591,7 @@ isexe@^2.0.0:
 
 js-yaml@^4.1.0:
   version "4.1.0"
-  resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz"
+  resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602"
   integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==
   dependencies:
     argparse "^2.0.1"
@@ -635,7 +640,7 @@ lodash.merge@^4.6.2:
 
 marked@^15.0.2:
   version "15.0.2"
-  resolved "https://registry.npmjs.org/marked/-/marked-15.0.2.tgz"
+  resolved "https://registry.yarnpkg.com/marked/-/marked-15.0.2.tgz#e6f8c351554e05cfbe77fee5fa2d6116d361787b"
   integrity sha512-85RUkoYKIVB21PbMKrnD6aCl9ws+XKEyhJNMbLn206NyD3jbBo7Ec7Wi4Jrsn4dV1a2ng7K/jfkmIN0DNoS41w==
 
 merge2@^1.3.0:
@@ -723,14 +728,19 @@ picomatch@^2.3.1:
   resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz"
   integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
 
+pluralize@^8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"
+  integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==
+
 prelude-ls@^1.2.1:
   version "1.2.1"
   resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz"
   integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==
 
 [email protected]:
   version "3.4.1"
-  resolved "https://registry.npmjs.org/prettier/-/prettier-3.4.1.tgz"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.4.1.tgz#e211d451d6452db0a291672ca9154bc8c2579f7b"
   integrity sha512-G+YdqtITVZmOJje6QkXQWzl3fSfMxFwm1tjTyo9exhkmWSqC4Yhd1+lug++IlR2mvRVAxEDDWYkQdeSztajqgg==
 
 punycode@^2.1.0:
@@ -798,7 +808,7 @@ to-regex-range@^5.0.1:
 
 trieve-ts-sdk@^0.0.35:
   version "0.0.35"
-  resolved "https://registry.npmjs.org/trieve-ts-sdk/-/trieve-ts-sdk-0.0.35.tgz"
+  resolved "https://registry.yarnpkg.com/trieve-ts-sdk/-/trieve-ts-sdk-0.0.35.tgz#c93f0dba6bd3422cdeae4e569b9dfa59688162f5"
   integrity sha512-aTaJ+CQDOvnef1Vt+d/tJX+Y3eRleWqxdlbn80VX3XJ7kUQZog37XdqepjqV0RHZt+8+NKMZ7nkUBXQwxPc/gw==
 
 ts-api-utils@^1.3.0:
@@ -815,7 +825,7 @@ type-check@^0.4.0, type-check@~0.4.0:
 
 typescript-eslint@^8.16.0:
   version "8.16.0"
-  resolved "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.16.0.tgz"
+  resolved "https://registry.yarnpkg.com/typescript-eslint/-/typescript-eslint-8.16.0.tgz#d608c972d6b2461ca10ec30fd3fa62a080baba19"
   integrity sha512-wDkVmlY6O2do4V+lZd0GtRfbtXbeD0q9WygwXXSJnC1xorE8eqyC2L1tJimqpSeFrOzRlYtWnUp/uzgHQOgfBQ==
   dependencies:
     "@typescript-eslint/eslint-plugin" "8.16.0"
@@ -841,12 +851,12 @@ uri-js@^4.2.2:
 
 webidl-conversions@^7.0.0:
   version "7.0.0"
-  resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz"
+  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a"
   integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==
 
 whatwg-mimetype@^3.0.0:
   version "3.0.0"
-  resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz"
+  resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7"
   integrity sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==
 
 which@^2.0.1:
@@ -863,7 +873,7 @@ word-wrap@^1.2.5:
 
 yaml@^2.6.1:
   version "2.6.1"
-  resolved "https://registry.npmjs.org/yaml/-/yaml-2.6.1.tgz"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.6.1.tgz#42f2b1ba89203f374609572d5349fb8686500773"
   integrity sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg==
 
 yocto-queue@^0.1.0: