Hook up frontend

packages/api/ai/generate.mts

@@ -13,7 +13,7 @@ import Path from 'node:path';
 import { PROMPTS_DIR } from '../constants.mjs';
 import { encode, decodeCells } from '../srcmd.mjs';
 import { buildProjectXml, type FileContent } from '../ai/app-parser.mjs';
-import { type AppGenerationLog, logAppGeneration } from './logger.mjs';
+import { logAppGeneration } from './logger.mjs';
 
 const makeGenerateSrcbookSystemPrompt = () => {
   return readFileSync(Path.join(PROMPTS_DIR, 'srcbook-generator.txt'), 'utf-8');
@@ -259,46 +259,38 @@ export async function generateApp(
   return result.text;
 }
 
-export async function editApp(
+export async function streamEditApp(
   projectId: string,
   files: FileContent[],
   query: string,
   appId: string,
   planId: string,
-): Promise<string> {
-  const model = await getModel();
-  const systemPrompt = makeAppEditorSystemPrompt();
-  const userPrompt = makeAppEditorUserPrompt(projectId, files, query);
-  const result = await generateText({
-    model,
-    system: systemPrompt,
-    prompt: userPrompt,
-  });
-  const log: AppGenerationLog = {
-    appId,
-    planId,
-    llm_request: { model, system: systemPrompt, prompt: userPrompt },
-    llm_response: result,
-  };
-
-  if (process.env.SRCBOOK_DISABLE_ANALYTICS !== 'true') {
-    logAppGeneration(log);
-  }
-  return result.text;
-}
-
-export async function streamEditApp(projectId: string, files: FileContent[], query: string) {
+) {
   const model = await getModel();
 
   const systemPrompt = makeAppEditorSystemPrompt();
   const userPrompt = makeAppEditorUserPrompt(projectId, files, query);
 
+  let response = '';
+
   const result = await streamText({
     model,
     system: systemPrompt,
     prompt: userPrompt,
+    onChunk: (chunk) => {
+      if (chunk.chunk.type === 'text-delta') {
+        response += chunk.chunk.textDelta;
+      }
+    },
     onFinish: () => {
-      // TODO: log the result
+      if (process.env.SRCBOOK_DISABLE_ANALYTICS !== 'true') {
+        logAppGeneration({
+          appId,
+          planId,
+          llm_request: { model, system: systemPrompt, prompt: userPrompt },
+          llm_response: response,
+        });
+      }
     },
   });
 

packages/api/ai/plan-parser.mts

@@ -1,9 +1,9 @@
-import fs from 'node:fs';
 import { XMLParser } from 'fast-xml-parser';
 import Path from 'node:path';
 import { type App as DBAppType } from '../db/schema.mjs';
 import { loadFile } from '../apps/disk.mjs';
 import { StreamingXMLParser, TagType } from './stream-xml-parser.mjs';
+import { ActionChunkType, DescriptionChunkType } from '@srcbook/shared';
 
 // The ai proposes a plan that we expect to contain both files and commands
 // Here is an example of a plan:
@@ -172,56 +172,32 @@ export function getPackagesToInstall(plan: Plan): string[] {
 
 export async function streamParsePlan(
   stream: AsyncIterable<string>,
-  _app: DBAppType,
+  app: DBAppType,
   _query: string,
-  _planId: string,
+  planId: string,
 ) {
   let parser: StreamingXMLParser;
 
-  const writeStream = fs.createWriteStream('/Users/ben/Desktop/out.txt');
-
   return new ReadableStream({
     async pull(controller) {
       if (parser === undefined) {
         parser = new StreamingXMLParser({
-          onTag(tag) {
-            try {
-              const chunk = toStreamingChunk(tag);
+          async onTag(tag) {
+            if (tag.name === 'planDescription' || tag.name === 'action') {
+              const chunk = await toStreamingChunk(app, tag, planId);
               if (chunk) {
                 controller.enqueue(JSON.stringify(chunk) + '\n');
               }
-            } catch (error) {
-              console.error(error);
-              controller.enqueue(
-                JSON.stringify({
-                  type: 'error',
-                  data: { content: 'Error while parsing streaming response' },
-                }) + '\n',
-              );
-              controller.error(error);
             }
           },
         });
       }
 
       try {
         for await (const chunk of stream) {
-          writeStream.write(JSON.stringify({ chunk }) + '\n');
-          try {
-            parser.parse(chunk);
-          } catch (error) {
-            console.error(error);
-            controller.enqueue(
-              JSON.stringify({
-                type: 'error',
-                data: { content: 'Error while parsing streaming response' },
-              }) + '\n',
-            );
-            controller.error(error);
-          }
+          parser.parse(chunk);
         }
         controller.close();
-        writeStream.end();
       } catch (error) {
         console.error(error);
         controller.enqueue(
@@ -236,15 +212,18 @@ export async function streamParsePlan(
   });
 }
 
-function toStreamingChunk(tag: TagType) {
-  console.log('TAG', tag);
-
+async function toStreamingChunk(
+  app: DBAppType,
+  tag: TagType,
+  planId: string,
+): Promise<DescriptionChunkType | ActionChunkType | null> {
   switch (tag.name) {
     case 'planDescription':
       return {
         type: 'description',
+        planId: planId,
         data: { content: tag.content },
-      };
+      } as DescriptionChunkType;
     case 'action': {
       const descriptionTag = tag.children.find((t) => t.name === 'description');
       const description = descriptionTag?.content ?? '';
@@ -253,33 +232,48 @@ function toStreamingChunk(tag: TagType) {
       if (type === 'file') {
         const fileTag = tag.children.find((t) => t.name === 'file')!;
 
+        const filePath = fileTag.attributes.filename as string;
+        let originalContent = null;
+
+        try {
+          const fileContent = await loadFile(app, filePath);
+          originalContent = fileContent.source;
+        } catch (error) {
+          // If the file doesn't exist, it's likely that it's a new file.
+        }
+
         return {
           type: 'action',
+          planId: planId,
           data: {
             type: 'file',
             description,
-            file: {
-              content: fileTag.content,
-              filename: fileTag.attributes.filename,
-            },
+            path: filePath,
+            dirname: Path.dirname(filePath),
+            basename: Path.basename(filePath),
+            modified: fileTag.content,
+            original: originalContent,
           },
-        };
+        } as ActionChunkType;
       } else if (type === 'command') {
         const commandTag = tag.children.find((t) => t.name === 'commandType')!;
         const packageTags = tag.children.filter((t) => t.name === 'package');
 
         return {
           type: 'action',
+          planId: planId,
           data: {
             type: 'command',
             description,
-            command: {
-              type: commandTag.content,
-              packages: packageTags.map((t) => t.content),
-            },
+            command: commandTag.content,
+            packages: packageTags.map((t) => t.content),
           },
-        };
+        } as ActionChunkType;
+      } else {
+        return null;
       }
     }
+    default:
+      return null;
   }
 }

packages/api/ai/stream-xml-parser.mts

@@ -57,9 +57,7 @@ export class StreamingXMLParser {
     if (!this.currentTag) return;
 
     if (this.currentTag.name === tagName) {
-      if (tagName === 'planDescription' || tagName === 'action') {
-        this.onTag(this.currentTag);
-      }
+      this.onTag(this.currentTag);
 
       if (this.tagStack.length > 0) {
         this.currentTag = this.tagStack.pop()!;

packages/api/server/http.mts

@@ -556,7 +556,7 @@ router.post('/apps/:id/edit', cors(), async (req, res) => {
     }
     const validName = toValidPackageName(app.name);
     const files = await getFlatFilesForApp(String(app.externalId));
-    const result = await streamEditApp(validName, files, query);
+    const result = await streamEditApp(validName, files, query, app.externalId, planId);
     const planStream = await streamParsePlan(result, app, query, planId);
 
     return streamJsonResponse(planStream, res, { status: 200 });

packages/api/test/streaming-xml-parser.test.mts

@@ -17,7 +17,9 @@ describe('parsePlan', () => {
     const tags: TagType[] = [];
     const parser = new StreamingXMLParser({
       onTag: (tag) => {
-        tags.push(tag);
+        if (tag.name === 'planDescription' || tag.name === 'action') {
+          tags.push(tag);
+        }
       },
     });
     getExampleChunks('../plan-chunks.txt').forEach((chunk) => parser.parse(chunk));
@@ -103,7 +105,9 @@ export const playlists: PlaylistItem[] = [
     const tags: TagType[] = [];
     const parser = new StreamingXMLParser({
       onTag: (tag) => {
-        tags.push(tag);
+        if (tag.name === 'planDescription' || tag.name === 'action') {
+          tags.push(tag);
+        }
       },
     });
     getExampleChunks('../plan-chunks-2.txt').forEach((chunk) => parser.parse(chunk));

packages/shared/src/types/history.mts

@@ -15,17 +15,14 @@ export type UserMessageType = {
   planId: string;
 };
 
-export type NpmInstallCommand = {
+export type CommandMessageType = {
   type: 'command';
+  planId: string;
   command: 'npm install';
   packages: string[];
   description: string;
 };
 
-export type CommandMessageType = NpmInstallCommand & {
-  planId: string;
-};
-
 export type DiffMessageType = {
   type: 'diff';
   planId: string;
@@ -42,3 +39,36 @@ export type PlanMessageType = {
 export type MessageType = UserMessageType | DiffMessageType | CommandMessageType | PlanMessageType;
 
 export type HistoryType = Array<MessageType>;
+
+//////////////////////////////////////////
+// When streaming file objects from LLM //
+//////////////////////////////////////////
+
+export type DescriptionChunkType = {
+  type: 'description';
+  planId: string;
+  data: { content: string };
+};
+
+export type FileActionChunkType = {
+  type: 'file';
+  description: string;
+  modified: string;
+  original: string | null;
+  basename: string;
+  dirname: string;
+  path: string;
+};
+
+export type CommandActionChunkType = {
+  type: 'command';
+  description: string;
+  command: 'npm install';
+  packages: string[];
+};
+
+export type ActionChunkType = {
+  type: 'action';
+  planId: string;
+  data: FileActionChunkType | CommandActionChunkType;
+};

packages/web/src/clients/http/apps.ts

@@ -1,6 +1,8 @@
 import type {
+  ActionChunkType,
   AppGenerationFeedbackType,
   AppType,
+  DescriptionChunkType,
   DirEntryType,
   FileEntryType,
   FileType,
@@ -234,7 +236,7 @@ export async function aiEditApp(
   id: string,
   query: string,
   planId: string,
-): Promise<AsyncIterable<string>> {
+): Promise<AsyncIterable<DescriptionChunkType | ActionChunkType>> {
   const response = await fetch(API_BASE_URL + `/apps/${id}/edit`, {
     method: 'POST',
     headers: { 'content-type': 'application/json' },
@@ -246,7 +248,21 @@ export async function aiEditApp(
     throw new Error('Request failed');
   }
 
-  return StreamToIterable(response.body!.pipeThrough(new TextDecoderStream()));
+  const JSONDecoder = new TransformStream<string, DescriptionChunkType | ActionChunkType>({
+    transform(chunk, controller) {
+      const lines = chunk.split('\n');
+      for (const line of lines) {
+        if (line.trim() !== '') {
+          const parsed = JSON.parse(line);
+          controller.enqueue(parsed);
+        }
+      }
+    },
+  });
+
+  return StreamToIterable(
+    response.body!.pipeThrough(new TextDecoderStream()).pipeThrough(JSONDecoder),
+  );
 }
 
 export async function loadHistory(id: string): Promise<{ data: HistoryType }> {