Merge pull request #47 from mattzcarey/refactor/memory-vector-store-a…

…bstraction

refactor: abstracting the creation of a vector store

src/common/model/createMemoryStore.test.ts

@@ -0,0 +1,28 @@
+import { MemoryVectorStore } from 'langchain/vectorstores/memory';
+import { CreateMemoryStore } from './createMemoryStore';
+import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
+import { initialFiles } from '../../testFiles/initialFilesExample'
+
+describe('CreateMemoryStore function', () => {
+  it('Checks that the CreateMemoryStore function returns a MemoryVectorStore object', () => {
+    const result = CreateMemoryStore(initialFiles);
+
+    expect(result).toBeInstanceOf(Promise<MemoryVectorStore>);
+  });
+
+  it('Checks if the function provides the required functionality', async () => {
+    const [result, expectedResult]  = await Promise.all([
+      CreateMemoryStore(initialFiles),
+      MemoryVectorStore.fromDocuments(initialFiles, new OpenAIEmbeddings(), {}),
+    ])
+
+    expect(result.memoryVectors.length).toEqual(expectedResult.memoryVectors.length);
+  });
+
+  it("Checks if the MemoryVectorStore returned returns a number in a similarity search", async () => {
+    const result = await CreateMemoryStore(initialFiles);
+    const ssWithScore = await result.similaritySearchWithScore("hello", 1);
+
+    expect(typeof ssWithScore[0][1] === 'number').toBe(true);
+  });
+});

src/common/model/createMemoryStore.ts

@@ -0,0 +1,13 @@
+import { Document } from 'langchain/dist/document';
+import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
+import { MemoryVectorStore } from 'langchain/vectorstores/memory';
+
+export const CreateMemoryStore = async (initialFiles: Document<Record<string, any>>[]): Promise<MemoryVectorStore> => {
+  const embeddingModel = new OpenAIEmbeddings();
+
+  return await MemoryVectorStore.fromDocuments(
+    initialFiles,
+    embeddingModel,
+    {}
+  );
+}

src/review/prompt/filterFiles/filterFiles.test.ts

@@ -28,10 +28,10 @@ describe("filterFiles unit test", () => {
     );
 
     const result = await filterFiles(testFiles);
+    const filesRegex = new RegExp(`(src/testFiles/longFile.tsx|src/testFiles/initialFilesExample.ts)`, "i");
 
-    expect(result.length).toEqual(1);
-    expect(result[0].fileName).toBe(
-      join(__dirname, "../../../testFiles", "longFile.tsx")
-    );
+    expect(result.length).toEqual(2);
+    expect(result[0].fileName).toMatch(filesRegex);
+    expect(result[1].fileName).toMatch(filesRegex);
   });
 });

src/review/prompt/makeSlimmedFile.ts

@@ -1,9 +1,8 @@
-import { OpenAIEmbeddings } from "langchain/embeddings/openai";
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
-import { MemoryVectorStore } from "langchain/vectorstores/memory";
 import { getLanguageOfFile } from "./getLanguageOfFile";
 import { slimmedContextPrompt } from "./prompts";
 import { ReviewFile } from "./types";
+import { CreateMemoryStore } from '../../common/model/createMemoryStore';
 import { File } from "../../common/types";
 import { logger } from "../../common/utils/logger";
 
@@ -36,10 +35,7 @@ export const makeSlimmedFile = async (
   const doc = await splitter.createDocuments([changedLines]);
 
   // Generate memory store with the whole file
-  const fileEmbeddings = await MemoryVectorStore.fromDocuments(
-    doc,
-    new OpenAIEmbeddings()
-  );
+  const fileEmbeddings = await CreateMemoryStore(doc);
 
   // Make a similarity search between the embeddings of the whole file
   // and the embeddings of the changed lines.

src/test/load/loadSnapshots.ts

@@ -3,6 +3,7 @@ import path from "path";
 import { TextLoader } from "langchain/document_loaders/fs/text";
 import { MemoryVectorStore } from "langchain/vectorstores/memory";
 import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+import { CreateMemoryStore } from '../../common/model/createMemoryStore';
 
 /**
  * Load a snapshot for a test from a file.
@@ -28,9 +29,6 @@ export const loadSnapshots = async (shapshotsDir: string) => {
       return loadSnapshot(path.join(shapshotsDir, snapshotFile));
     })
   );
-  return MemoryVectorStore.fromDocuments(
-    snapshots.flat(),
-    new OpenAIEmbeddings(),
-    {}
-  );
+
+  return await CreateMemoryStore(snapshots.flat());
 };

src/testFiles/initialFilesExample.ts

@@ -0,0 +1,50 @@
+export const initialFiles = [
+  {
+    pageContent: '**LOGAF Level 1 - src/test/cases/.cache/faee919bf4f6a5b85a44b1a8eacc0ca24223d6c4033a2b4c52bc79bb8e1bc1bb.ts**\n' +
+      '\n' +
+      'The code exposes a secret key which is a serious security issue. Never log sensitive information like API keys, passwords, or secrets. Consider using environment variables to store such sensitive information. For example:\n' +
+      '\n' +
+      '```typescript\n' +
+      'const secretKey = process.env.SECRET_KEY;\n' +
+      '\n' +
+      'function exposeSecret() {\n' +
+      '  console.log(`The secret key is: ${secretKey}`);\n' +
+      '}\n' +
+      '\n' +
+      'exposeSecret();\n' +
+      '```\n' +
+      '\n' +
+      'In this case, the secret key is stored in an environment variable named `SECRET_KEY`. Remember to add `SECRET_KEY` to your `.env` file and never commit the `.env` file to the repository.\n' +
+      '\n' +
+      '🔑❌🔒\n',
+    metadata: {
+      source: '/Users/sebo/Desktop/aleios/code-review-gpt/src/test/cases/snapshots/exposed-secret.md'
+    }
+  },
+  {
+    pageContent: '**LOGAF Level 1 - src/test/cases/.cache/5519e4e1b45143a504ec259a5d911dea930372c19b3f56b51afab53f55339b56.ts**\n' +
+      '\n' +
+      'The function `nestedLoops` has too many nested loops which can lead to performance issues and is hard to read and maintain. Consider refactoring the code to reduce the number of nested loops. If the logic of the code allows, you could use recursion or divide the task into smaller functions. Here is an example of how you could refactor this code using recursion:\n' +
+      '\n' +
+      '```\n' +
+      'function recursiveLoop(depth, maxDepth, maxCount) {\n' +
+      '    if (depth === maxDepth) {\n' +
+      '        console.log(...arguments);\n' +
+      '    } else {\n' +
+      '        for (let i = 0; i < maxCount; i++) {\n' +
+      '            recursiveLoop(i, depth + 1, maxDepth, maxCount);\n' +
+      '        }\n' +
+      '    }\n' +
+      '}\n' +
+      '\n' +
+      'recursiveLoop(0, 10, 10);\n' +
+      '```\n' +
+      '\n' +
+      'This code does the same thing as the original code but is much easier to read and maintain.\n' +
+      '\n' +
+      '🔄🐌🔧\n',
+    metadata: {
+      source: '/Users/sebo/Desktop/aleios/code-review-gpt/src/test/cases/snapshots/too-many-nested-loops.md'
+    }
+  },
+];