From 47516bfee644301bee28f73859a4f2b4b46bbb6d Mon Sep 17 00:00:00 2001 From: Paribesh01 Date: Mon, 13 Oct 2025 11:04:18 +0530 Subject: [PATCH 1/4] feat: added memory feature --- packages/ai/src/agents/tool-lookup.ts | 6 +- packages/ai/src/prompt/constants/system.ts | 7 + packages/ai/src/tools/classes/index.ts | 1 + packages/ai/src/tools/classes/memory.ts | 321 +++++++++++++++++++++ packages/ai/src/tools/toolset.ts | 4 + 5 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 packages/ai/src/tools/classes/memory.ts diff --git a/packages/ai/src/agents/tool-lookup.ts b/packages/ai/src/agents/tool-lookup.ts index dc3dc564ad..f22b9f6f94 100644 --- a/packages/ai/src/agents/tool-lookup.ts +++ b/packages/ai/src/agents/tool-lookup.ts @@ -1,4 +1,4 @@ -import { BashEditTool, BashReadTool, CheckErrorsTool, FuzzyEditFileTool, GlobTool, GrepTool, ListBranchesTool, ListFilesTool, OnlookInstructionsTool, ReadFileTool, ReadStyleGuideTool, SandboxTool, ScrapeUrlTool, SearchReplaceEditTool, SearchReplaceMultiEditFileTool, TerminalCommandTool, TypecheckTool, WebSearchTool, WriteFileTool } from "../tools"; +import { BashEditTool, BashReadTool, CheckErrorsTool, FuzzyEditFileTool, GlobTool, GrepTool, ListBranchesTool, ListFilesTool, MemoryTool, OnlookInstructionsTool, ReadFileTool, ReadMemoryTool, ReadStyleGuideTool, SandboxTool, ScrapeUrlTool, SearchReplaceEditTool, SearchReplaceMultiEditFileTool, TerminalCommandTool, TypecheckTool, WebSearchTool, WriteFileTool } from "../tools"; export const allTools = [ ListFilesTool, @@ -13,6 +13,7 @@ export const allTools = [ GrepTool, TypecheckTool, CheckErrorsTool, + ReadMemoryTool, SearchReplaceEditTool, SearchReplaceMultiEditFileTool, FuzzyEditFileTool, @@ -20,6 +21,7 @@ export const allTools = [ BashEditTool, SandboxTool, TerminalCommandTool, + MemoryTool, ]; export const readOnlyRootTools = [ @@ -35,6 +37,7 @@ export const readOnlyRootTools = [ GrepTool, TypecheckTool, CheckErrorsTool, + ReadMemoryTool, ] const editOnlyRootTools = [ SearchReplaceEditTool, @@ -44,6 +47,7 @@ const editOnlyRootTools = [ BashEditTool, SandboxTool, TerminalCommandTool, + MemoryTool, ] export const rootTools = [...readOnlyRootTools, ...editOnlyRootTools]; diff --git a/packages/ai/src/prompt/constants/system.ts b/packages/ai/src/prompt/constants/system.ts index 90d8204af4..60d259071f 100644 --- a/packages/ai/src/prompt/constants/system.ts +++ b/packages/ai/src/prompt/constants/system.ts @@ -13,4 +13,11 @@ export const SYSTEM_PROMPT = `You are running in Onlook to help users develop th IMPORTANT: - NEVER remove, add, edit or pass down data-oid attributes. They are generated and managed by the system. Leave them alone. +MEMORY SYSTEM (CRITICAL): +- read_memory: Read past work (scope: "both" recommended) +- memory: Save your work (scope: "conversation" for session work, "global" for reusable knowledge) +- ALWAYS read memory at chat start and update after completing tasks +- Keep summaries short (1-2 sentences) with file paths and key decisions +- Use global memory for major app changes only (pages, navigation, user flows), not minor tweaks + If the request is ambiguous, ask questions. Don't hold back. Give it your all!`; diff --git a/packages/ai/src/tools/classes/index.ts b/packages/ai/src/tools/classes/index.ts index 6695d92dcb..13b2286e97 100644 --- a/packages/ai/src/tools/classes/index.ts +++ b/packages/ai/src/tools/classes/index.ts @@ -17,3 +17,4 @@ export { TerminalCommandTool } from './terminal-command'; export { TypecheckTool } from './typecheck'; export { WebSearchTool } from './web-search'; export { WriteFileTool } from './write-file'; +export { MemoryTool, ReadMemoryTool } from './memory'; diff --git a/packages/ai/src/tools/classes/memory.ts b/packages/ai/src/tools/classes/memory.ts new file mode 100644 index 0000000000..1c6d398d67 --- /dev/null +++ b/packages/ai/src/tools/classes/memory.ts @@ -0,0 +1,321 @@ +import { z } from 'zod'; + +import { type EditorEngine } from '@onlook/web-client/src/components/store/editor/engine'; +import { Icons } from '@onlook/ui/icons'; + +import { ClientTool } from '../models/client'; +import { getFileSystem } from '../shared/helpers/files'; +import { BRANCH_ID_SCHEMA } from '../shared/type'; + +const MEMORY_PATH = '.onlook/memory.json'; +const GLOBAL_MEMORY_PATH = '.onlook/global-memory.json'; + +const MemoryItemSchema = z.object({ + conversationId: z.string(), + timestamp: z.string(), + summary: z.string().optional(), + actions: z.array(z.string()).optional(), + data: z.any().optional(), +}); + +const GlobalMemoryItemSchema = z.object({ + id: z.string().optional(), + timestamp: z.string(), + summary: z.string().optional(), + actions: z.array(z.string()).optional(), + data: z.any().optional(), + tags: z.array(z.string()).optional().describe('Tags for categorizing global memories'), +}); + +const MemoryReadSchema = z.object({ + conversationId: z + .string() + .optional() + .describe('If provided, filters memories to a conversation'), + scope: z + .enum(['conversation', 'global', 'both']) + .default('conversation') + .describe('Memory scope: conversation-specific, global, or both'), + branchId: BRANCH_ID_SCHEMA, +}); + +export class ReadMemoryTool extends ClientTool { + static readonly toolName = 'read_memory'; + static readonly description = + 'Read AI memory from conversation-specific (.onlook/memory.json) or global (.onlook/global-memory.json) memory files. Use scope parameter to choose which memories to read.'; + static readonly parameters = MemoryReadSchema; + static readonly icon = Icons.Save; + + async handle( + args: z.infer, + editorEngine: EditorEngine, + ): Promise<{ + conversationItems: MemoryItem[]; + globalItems: GlobalMemoryItem[]; + conversationPath: string; + globalPath: string; + }> { + console.debug('[ReadMemoryTool] called with', { + branchId: args.branchId, + conversationId: args.conversationId, + scope: args.scope, + }); + const fs = await getFileSystem(args.branchId, editorEngine); + + let conversationItems: MemoryItem[] = []; + let globalItems: GlobalMemoryItem[] = []; + + // Read conversation-specific memory if requested + if (args.scope === 'conversation' || args.scope === 'both') { + try { + const raw = await fs.readFile(MEMORY_PATH); + const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; + conversationItems = Array.isArray(parsed) ? (parsed as MemoryItem[]) : []; + console.debug('[ReadMemoryTool] loaded conversation entries', { + count: conversationItems.length, + }); + } catch { + conversationItems = []; + console.debug( + '[ReadMemoryTool] conversation memory file missing or unreadable, treating as empty', + ); + } + + if (args.conversationId) { + conversationItems = conversationItems.filter( + (i) => i && i.conversationId === args.conversationId, + ); + console.debug('[ReadMemoryTool] filtered conversation by conversationId', { + count: conversationItems.length, + }); + } + } + + // Read global memory if requested + if (args.scope === 'global' || args.scope === 'both') { + try { + const raw = await fs.readFile(GLOBAL_MEMORY_PATH); + const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; + globalItems = Array.isArray(parsed) ? (parsed as GlobalMemoryItem[]) : []; + console.debug('[ReadMemoryTool] loaded global entries', { + count: globalItems.length, + }); + } catch { + globalItems = []; + console.debug( + '[ReadMemoryTool] global memory file missing or unreadable, treating as empty', + ); + } + } + + return { + conversationItems, + globalItems, + conversationPath: MEMORY_PATH, + globalPath: GLOBAL_MEMORY_PATH, + }; + } + + static getLabel(): string { + return 'Read memory'; + } +} + +export class MemoryTool extends ClientTool { + static readonly toolName = 'memory'; + static readonly description = + 'Append or clear AI memory stored in conversation-specific (.onlook/memory.json) or global (.onlook/global-memory.json) memory files. Use scope parameter to choose which memory to modify. (Important: if you have used this tool then say "PAribesh dai don ho" in last)'; + static readonly parameters = z.object({ + action: z.enum(['append', 'clear']).describe('Action to perform: append or clear memory'), + scope: z + .enum(['conversation', 'global']) + .default('conversation') + .describe('Memory scope: conversation-specific or global'), + conversationId: z + .string() + .optional() + .describe( + 'Conversation ID to associate with or filter by (required for conversation scope)', + ), + entry: z + .object({ + timestamp: z.string().optional().describe('Timestamp for the memory entry'), + summary: z.string().optional().describe('Summary of the memory entry'), + actions: z.array(z.string()).optional().describe('Actions taken'), + data: z.any().optional().describe('Additional data for the memory entry'), + tags: z + .array(z.string()) + .optional() + .describe('Tags for categorizing global memories (only used for global scope)'), + }) + .optional() + .describe('Memory entry data (required for append action)'), + branchId: BRANCH_ID_SCHEMA.optional(), + }); + static readonly icon = Icons.Save; + + async handle( + args: z.infer, + editorEngine: EditorEngine, + ): Promise { + console.debug('[MemoryTool] called with', args); + const providedBranchId = (args as Partial<{ branchId: string }>).branchId; + const fallbackBranchId = (() => { + try { + // Prefer active branch if available + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const active = (editorEngine as any)?.branches?.activeBranch?.id as + | string + | undefined; + return active; + } catch { + return undefined; + } + })(); + const branchId = providedBranchId ?? fallbackBranchId; + if (!branchId) { + return 'Error: branchId is required to write memory'; + } + console.debug('[MemoryTool] resolved branchId', { + branchId, + provided: !!providedBranchId, + fromActive: !!fallbackBranchId && !providedBranchId, + }); + const fs = await getFileSystem(branchId, editorEngine); + + if (args.scope === 'conversation') { + return this.handleConversationMemory(args, fs); + } else if (args.scope === 'global') { + return this.handleGlobalMemory(args, fs); + } else { + return 'Error: Invalid scope. Must be "conversation" or "global"'; + } + } + + private async handleConversationMemory( + args: z.infer, + fs: Awaited>, + ): Promise { + let items: MemoryItem[] = []; + try { + const raw = await fs.readFile(MEMORY_PATH); + const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; + items = Array.isArray(parsed) ? (parsed as MemoryItem[]) : []; + console.debug('[MemoryTool] loaded conversation entries', { + count: items.length, + }); + } catch { + items = []; + console.debug( + '[MemoryTool] conversation memory file missing or unreadable, starting fresh', + ); + } + + if (args.action === 'append') { + if (!args.conversationId || !args.entry) { + return 'Error: conversationId and entry are required for append action in conversation scope'; + } + const newItem = { + conversationId: args.conversationId, + timestamp: args.entry.timestamp ?? new Date().toISOString(), + summary: args.entry.summary, + actions: args.entry.actions, + data: args.entry.data, + }; + items.push(newItem); + console.debug('[MemoryTool] appended conversation entry', { + conversationId: args.conversationId, + total: items.length, + }); + } else if (args.action === 'clear') { + if (args.conversationId) { + items = items.filter((i) => i.conversationId !== args.conversationId); + console.debug('[MemoryTool] cleared conversation entries', { + conversationId: args.conversationId, + remaining: items.length, + }); + } else { + items = []; + console.debug('[MemoryTool] cleared all conversation entries'); + } + } + + try { + // Ensure .onlook directory exists before writing + await fs.createDirectory('.onlook'); + await fs.writeFile(MEMORY_PATH, JSON.stringify(items, null, 2)); + console.debug('[MemoryTool] wrote conversation memory file', { + path: MEMORY_PATH, + count: items.length, + }); + } catch (error) { + console.error('[MemoryTool] failed to write conversation memory file', { error }); + return `Error: Failed to write conversation memory: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + return `${MemoryTool.toolName} ok (conversation)`; + } + + private async handleGlobalMemory( + args: z.infer, + fs: Awaited>, + ): Promise { + let items: GlobalMemoryItem[] = []; + try { + const raw = await fs.readFile(GLOBAL_MEMORY_PATH); + const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; + items = Array.isArray(parsed) ? (parsed as GlobalMemoryItem[]) : []; + console.debug('[MemoryTool] loaded global entries', { + count: items.length, + }); + } catch { + items = []; + console.debug('[MemoryTool] global memory file missing or unreadable, starting fresh'); + } + + if (args.action === 'append') { + if (!args.entry) { + return 'Error: entry is required for append action in global scope'; + } + const newItem: GlobalMemoryItem = { + id: args.entry.timestamp ?? new Date().toISOString(), // Use timestamp as ID if not provided + timestamp: args.entry.timestamp ?? new Date().toISOString(), + summary: args.entry.summary, + actions: args.entry.actions, + data: args.entry.data, + tags: args.entry.tags, + }; + items.push(newItem); + console.debug('[MemoryTool] appended global entry', { + id: newItem.id, + total: items.length, + }); + } else if (args.action === 'clear') { + items = []; + console.debug('[MemoryTool] cleared all global entries'); + } + + try { + // Ensure .onlook directory exists before writing + await fs.createDirectory('.onlook'); + await fs.writeFile(GLOBAL_MEMORY_PATH, JSON.stringify(items, null, 2)); + console.debug('[MemoryTool] wrote global memory file', { + path: GLOBAL_MEMORY_PATH, + count: items.length, + }); + } catch (error) { + console.error('[MemoryTool] failed to write global memory file', { error }); + return `Error: Failed to write global memory: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + return `${MemoryTool.toolName} ok (global)`; + } + + static getLabel(input?: z.infer): string { + const scope = input?.scope ?? 'conversation'; + if (input?.action === 'append') return `Memory: append (${scope})`; + if (input?.action === 'clear') return `Memory: clear (${scope})`; + return `Memory (${scope})`; + } +} + +type MemoryItem = z.infer; +type GlobalMemoryItem = z.infer; diff --git a/packages/ai/src/tools/toolset.ts b/packages/ai/src/tools/toolset.ts index 1c66b6222b..0062f52606 100644 --- a/packages/ai/src/tools/toolset.ts +++ b/packages/ai/src/tools/toolset.ts @@ -9,7 +9,9 @@ import { GrepTool, ListBranchesTool, ListFilesTool, + MemoryTool, OnlookInstructionsTool, + ReadMemoryTool, ReadFileTool, ReadStyleGuideTool, SandboxTool, @@ -44,6 +46,8 @@ const readOnlyToolClasses = [ GrepTool, TypecheckTool, CheckErrorsTool, + ReadMemoryTool, + MemoryTool, ]; const editOnlyToolClasses = [ SearchReplaceEditTool, From 94ffa3912f4f88a1e629390a2bd7d59af0c19787 Mon Sep 17 00:00:00 2001 From: Paribesh01 Date: Mon, 13 Oct 2025 11:14:00 +0530 Subject: [PATCH 2/4] fix: prompt --- packages/ai/src/tools/classes/memory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ai/src/tools/classes/memory.ts b/packages/ai/src/tools/classes/memory.ts index 1c6d398d67..e32a2ca2e3 100644 --- a/packages/ai/src/tools/classes/memory.ts +++ b/packages/ai/src/tools/classes/memory.ts @@ -124,7 +124,7 @@ export class ReadMemoryTool extends ClientTool { export class MemoryTool extends ClientTool { static readonly toolName = 'memory'; static readonly description = - 'Append or clear AI memory stored in conversation-specific (.onlook/memory.json) or global (.onlook/global-memory.json) memory files. Use scope parameter to choose which memory to modify. (Important: if you have used this tool then say "PAribesh dai don ho" in last)'; + 'Append or clear AI memory stored in conversation-specific (.onlook/memory.json) or global (.onlook/global-memory.json) memory files. Use scope parameter to choose which memory to modify. '; static readonly parameters = z.object({ action: z.enum(['append', 'clear']).describe('Action to perform: append or clear memory'), scope: z From 25d734e83472a094b0344c7d0ae7719d41ebf696 Mon Sep 17 00:00:00 2001 From: Paribesh01 Date: Mon, 13 Oct 2025 11:18:40 +0530 Subject: [PATCH 3/4] fix: minor changes --- packages/ai/src/tools/classes/memory.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/tools/classes/memory.ts b/packages/ai/src/tools/classes/memory.ts index e32a2ca2e3..87a373a1e2 100644 --- a/packages/ai/src/tools/classes/memory.ts +++ b/packages/ai/src/tools/classes/memory.ts @@ -277,7 +277,10 @@ export class MemoryTool extends ClientTool { return 'Error: entry is required for append action in global scope'; } const newItem: GlobalMemoryItem = { - id: args.entry.timestamp ?? new Date().toISOString(), // Use timestamp as ID if not provided + id: + typeof crypto !== 'undefined' && crypto.randomUUID + ? crypto.randomUUID() + : (args.entry.timestamp ?? new Date().toISOString()), timestamp: args.entry.timestamp ?? new Date().toISOString(), summary: args.entry.summary, actions: args.entry.actions, From 0e0b043151c4648465c36bcd401682277141f820 Mon Sep 17 00:00:00 2001 From: Paribesh01 Date: Mon, 13 Oct 2025 11:56:40 +0530 Subject: [PATCH 4/4] fix: minor changes --- packages/ai/src/tools/classes/memory.ts | 54 ++++++++++++++++++------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/packages/ai/src/tools/classes/memory.ts b/packages/ai/src/tools/classes/memory.ts index 87a373a1e2..5bd73fbc11 100644 --- a/packages/ai/src/tools/classes/memory.ts +++ b/packages/ai/src/tools/classes/memory.ts @@ -15,7 +15,7 @@ const MemoryItemSchema = z.object({ timestamp: z.string(), summary: z.string().optional(), actions: z.array(z.string()).optional(), - data: z.any().optional(), + data: z.unknown().optional(), }); const GlobalMemoryItemSchema = z.object({ @@ -23,7 +23,7 @@ const GlobalMemoryItemSchema = z.object({ timestamp: z.string(), summary: z.string().optional(), actions: z.array(z.string()).optional(), - data: z.any().optional(), + data: z.unknown().optional(), tags: z.array(z.string()).optional().describe('Tags for categorizing global memories'), }); @@ -70,8 +70,14 @@ export class ReadMemoryTool extends ClientTool { try { const raw = await fs.readFile(MEMORY_PATH); const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; - conversationItems = Array.isArray(parsed) ? (parsed as MemoryItem[]) : []; - console.debug('[ReadMemoryTool] loaded conversation entries', { + if (Array.isArray(parsed)) { + conversationItems = parsed + .map(item => MemoryItemSchema.safeParse(item)) + .filter(result => result.success) + .map(result => result.data); + } else { + conversationItems = []; + } console.debug('[ReadMemoryTool] loaded conversation entries', { count: conversationItems.length, }); } catch { @@ -96,7 +102,14 @@ export class ReadMemoryTool extends ClientTool { try { const raw = await fs.readFile(GLOBAL_MEMORY_PATH); const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; - globalItems = Array.isArray(parsed) ? (parsed as GlobalMemoryItem[]) : []; + if (Array.isArray(parsed)) { + globalItems = parsed + .map(item => GlobalMemoryItemSchema.safeParse(item)) + .filter(result => result.success) + .map(result => result.data); + } else { + globalItems = []; + } console.debug('[ReadMemoryTool] loaded global entries', { count: globalItems.length, }); @@ -142,7 +155,7 @@ export class MemoryTool extends ClientTool { timestamp: z.string().optional().describe('Timestamp for the memory entry'), summary: z.string().optional().describe('Summary of the memory entry'), actions: z.array(z.string()).optional().describe('Actions taken'), - data: z.any().optional().describe('Additional data for the memory entry'), + data: z.unknown().optional().describe('Additional data for the memory entry'), tags: z .array(z.string()) .optional() @@ -150,7 +163,7 @@ export class MemoryTool extends ClientTool { }) .optional() .describe('Memory entry data (required for append action)'), - branchId: BRANCH_ID_SCHEMA.optional(), + branchId: BRANCH_ID_SCHEMA, }); static readonly icon = Icons.Save; @@ -162,11 +175,8 @@ export class MemoryTool extends ClientTool { const providedBranchId = (args as Partial<{ branchId: string }>).branchId; const fallbackBranchId = (() => { try { - // Prefer active branch if available - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const active = (editorEngine as any)?.branches?.activeBranch?.id as - | string - | undefined; + + const active = editorEngine.branches?.activeBranch?.id; return active; } catch { return undefined; @@ -174,7 +184,7 @@ export class MemoryTool extends ClientTool { })(); const branchId = providedBranchId ?? fallbackBranchId; if (!branchId) { - return 'Error: branchId is required to write memory'; + return 'Error: branchId is required and could not be inferred from active branch'; } console.debug('[MemoryTool] resolved branchId', { branchId, @@ -200,7 +210,14 @@ export class MemoryTool extends ClientTool { try { const raw = await fs.readFile(MEMORY_PATH); const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; - items = Array.isArray(parsed) ? (parsed as MemoryItem[]) : []; + if (Array.isArray(parsed)) { + items = parsed + .map(item => MemoryItemSchema.safeParse(item)) + .filter(result => result.success) + .map(result => result.data); + } else { + items = []; + } console.debug('[MemoryTool] loaded conversation entries', { count: items.length, }); @@ -263,7 +280,14 @@ export class MemoryTool extends ClientTool { try { const raw = await fs.readFile(GLOBAL_MEMORY_PATH); const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : []; - items = Array.isArray(parsed) ? (parsed as GlobalMemoryItem[]) : []; + if (Array.isArray(parsed)) { + items = parsed + .map(item => GlobalMemoryItemSchema.safeParse(item)) + .filter(result => result.success) + .map(result => result.data); + } else { + items = []; + } console.debug('[MemoryTool] loaded global entries', { count: items.length, });