docs: add docs for how to use CLI

devflowinc · Dec 10, 2024 · 856dbb2 · 856dbb2
1 parent 20be69e
commit 856dbb2
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 15 deletions.
diff --git a/clients/trieve-fern-adapter/README.md b/clients/trieve-fern-adapter/README.md
@@ -2,13 +2,36 @@
 
 The easiest way to use Trieve in combination with Fern.
 
-## Installation
+## Collect all the data
+
+### Trieve
+
+You need to retrieve your `Org ID` and a new `API Key`:
+
+1. Go to the [Trieve dashboard](https://dashboard.trieve.ai/org).
+2. Copy the `Org ID`, you will need it later.
+3. Press on the `API Keys` tab.
+4. Press `Create New Key` and select `OWNER` for the permission.
+5. Copy the `API Key`, you will need it later.
+
+### Documents
+
+You need:
+
+- the path to your `<docs.yml>`, e.g. `./fern/docs.yml`.
+- (optional): the url to your `<openapi.json>`, e.g. `https://api.vapi.ai/api-json`.
+- (optional): the `<root url>` to your docs, e.g. `https://docs.vapi.ai`.
+- (optional): the `<api-reference-path>`, e.g. `api-reference`.
+
+## Local Setup
+
+### Installation
 
 ```bash
 npm install -g trieve-fern-adapter
 ```
 
-## Setup
+### Environment
 
 You need these environment variables set:
 
@@ -19,12 +42,65 @@ TRIEVE_ORGANIZATION_ID=
 TRIEVE_DATASET_TRACKING_ID=
 ```
 
-## How to use
+The `TRIEVE_DATASET_TRACKING_ID` must be an unique identifier for the dataset, e.g. `vapi`.
+
+### Execution
 
 ```bash
-trieve-fern-adapter --file <path-to-docs.yml> -s <openapi-spec-url> -r <root-url> -a <api-reference-page>
+trieve-fern-adapter --file <docs.yml> -s <openapi.json> -r <root-url> -a <api-reference-page>
+```
+
+## CI Setup (GitHub)
+
+### Environment
+
+Set these repository secrets:
+
+```
+TRIEVE_API_HOST=https://api.trieve.ai
+TRIEVE_API_KEY=
+TRIEVE_ORGANIZATION_ID=
+TRIEVE_DATASET_TRACKING_ID=
 ```
 
+### Workflow
+
+Add this workflow to `.github/workflows`.
+
+```yml
+name: Update Trieve
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Trieve Fern Adapter
+        run: npm install -g trieve-fern-adapter
+
+      - name: Update Trieve Chunks 
+        env:
+          TRIEVE_API_HOST: ${{ secrets.TRIEVE_API_HOST }}
+          TRIEVE_API_KEY: ${{ secrets.TRIEVE_API_KEY }}
+          TRIEVE_ORGANIZATION_ID: ${{ secrets.TRIEVE_ORGANIZATION_ID }}
+          TRIEVE_DATASET_TRACKING_ID: ${{ secrets.TRIEVE_DATASET_TRACKING_ID }}
+        run: trieve-fern-adapter --file <docs.yml> -s <openapi.json> -r <root-url> -a <api-reference-page>
+
+```
+
+Replace `<docs.yml>`, `<openapi.json>`, `<root-url>`, and `<api-reference-page>`.
+
 ## License
 
 MIT
diff --git a/clients/trieve-fern-adapter/package.json b/clients/trieve-fern-adapter/package.json
@@ -1,6 +1,6 @@
 {
   "name": "trieve-fern-adapter",
-  "version": "0.0.0",
+  "version": "0.0.3",
   "type": "module",
   "main": "dist/index.js",
   "bin": "dist/index.js",
@@ -41,7 +41,7 @@
     "happy-dom": "^15.11.6",
     "marked": "^15.0.2",
     "pluralize": "^8.0.0",
-    "trieve-ts-sdk": "^0.0.35",
+    "trieve-ts-sdk": "^0.0.40",
     "yaml": "^2.6.1"
   }
 }
diff --git a/clients/trieve-fern-adapter/src/index.ts b/clients/trieve-fern-adapter/src/index.ts
@@ -9,7 +9,7 @@ import { join } from 'node:path';
 import { TrieveSDK, ChunkReqPayload } from 'trieve-ts-sdk';
 import { marked } from 'marked';
 import { dereferenceSync } from '@trojs/openapi-dereference';
-import pluralize from 'pluralize'
+import pluralize from 'pluralize';
 
 const splitHtmlIntoHeadAndBodies = (html: Element): [string, string][] => {
   const headingRegex = /h\d/gi;
@@ -200,8 +200,10 @@ const extractChunksFromOpenapiSpec = async (
         const operationId = pathData[method].operationId;
         const summary = pathData[method].summary;
         const description = pathData[method].description;
-        const [namespace, ...parts] = summary?.toLowerCase().split(' ') ?? []
-        const endpoint = namespace ? join(pluralize(parts.join('-')), namespace) : path
+        const [namespace, ...parts] = summary?.toLowerCase().split(' ') ?? [];
+        const endpoint = namespace
+          ? join(pluralize(parts.join('-')), namespace)
+          : path;
         const pageLink = `${siteUrl}/${apiRefParent}/${endpoint}`;
         const metadata = {
           operation_id: operationId,
@@ -361,15 +363,23 @@ try {
 for (let i = 0; i < chunkReqPayloads.length; i += 120) {
   const chunkBatch = chunkReqPayloads.slice(i, i + 120);
   console.log(`Creating chunk batch ${i + 1} - ${i + 120}`);
+  let retries = 0;
   while (true) {
     try {
       await trieve.createChunk(chunkBatch);
+      console.log('Batch created');
       break;
     } catch (err) {
-      console.error('Error creating chunk batch, retrying...', err);
+      console.error('Error creating chunk batch...', err);
+      retries++;
+      if (retries > 3) {
+        console.error('Max retries exceeded, skipping batch');
+        break;
+      }
+      console.log('Retrying...');
     }
   }
 }
 
 console.log('Done!');
-process.exit(0);
+process.exit(0);
diff --git a/clients/trieve-fern-adapter/yarn.lock b/clients/trieve-fern-adapter/yarn.lock
@@ -806,10 +806,10 @@ to-regex-range@^5.0.1:
   dependencies:
     is-number "^7.0.0"
 
-trieve-ts-sdk@^0.0.35:
-  version "0.0.35"
-  resolved "https://registry.yarnpkg.com/trieve-ts-sdk/-/trieve-ts-sdk-0.0.35.tgz#c93f0dba6bd3422cdeae4e569b9dfa59688162f5"
-  integrity sha512-aTaJ+CQDOvnef1Vt+d/tJX+Y3eRleWqxdlbn80VX3XJ7kUQZog37XdqepjqV0RHZt+8+NKMZ7nkUBXQwxPc/gw==
+trieve-ts-sdk@^0.0.40:
+  version "0.0.40"
+  resolved "https://registry.yarnpkg.com/trieve-ts-sdk/-/trieve-ts-sdk-0.0.40.tgz#2214cdebd0e813e3c421496bd6a2eda7b2df4860"
+  integrity sha512-ageByWEepZj0DJly9D4ivNtLumwWoqX7HTub0B/FDqKgInWj4DsiScnFGGZp/rCuiHmSnC7VjKdPhbjLO32mYw==
 
 ts-api-utils@^1.3.0:
   version "1.4.2"