From d8ecac814ad2e3541e42955e0518d9ac312658b1 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Wed, 21 Jan 2026 16:15:17 -0500 Subject: [PATCH 01/13] Native images FP --- ui/desktop/src/components/BaseChat.tsx | 3 +- ui/desktop/src/components/ChatInput.tsx | 266 ++++++++++++++++++++---- ui/desktop/src/hooks/useChatStream.ts | 8 +- ui/desktop/src/types/message.ts | 27 ++- 4 files changed, 253 insertions(+), 51 deletions(-) diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 944fc25cb9a..4f688a234f9 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -162,11 +162,12 @@ function BaseChatContent({ const handleFormSubmit = (e: React.FormEvent) => { const customEvent = e as unknown as CustomEvent; const textValue = customEvent.detail?.value || ''; + const images = customEvent.detail?.images as import('../types/message').ImageData[] | undefined; if (recipe && textValue.trim()) { setHasStartedUsingRecipe(true); } - handleSubmit(textValue); + handleSubmit(textValue, images); }; const { sessionCosts } = useCostTracking({ diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index a91e4507b4a..f154b01b813 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -659,6 +659,55 @@ export default function ChatInput({ })); }; + // Helper function to compress/resize image + // Always resize to max 1024px on longest side and compress as JPEG at 0.85 quality + const compressImageDataUrl = async (dataUrl: string): Promise => { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const maxDimension = 1024; + let width = img.width; + let height = img.height; + + // Calculate new dimensions maintaining aspect ratio + if (width > maxDimension || height > maxDimension) { + if (width > height) { + height = Math.floor((height * maxDimension) / width); + width = maxDimension; + } else { + width = Math.floor((width * maxDimension) / height); + height = maxDimension; + } + } + + // Create canvas and resize + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + + if (!ctx) { + reject(new Error('Failed to get canvas context')); + return; + } + + ctx.drawImage(img, 0, 0, width, height); + + // Convert to JPEG with 0.85 quality + const compressedDataUrl = canvas.toDataURL('image/jpeg', 0.85); + const compressedSizeMB = (compressedDataUrl.length * 3) / 4 / (1024 * 1024); + + if (compressedSizeMB > MAX_IMAGE_SIZE_MB) { + reject(new Error(`Image is ${Math.round(compressedSizeMB)}MB after compression, exceeds ${MAX_IMAGE_SIZE_MB}MB limit`)); + } else { + resolve(compressedDataUrl); + } + }; + img.onerror = () => reject(new Error('Failed to load image for compression')); + img.src = dataUrl; + }); + }; + const handlePaste = async (evt: React.ClipboardEvent) => { const files = Array.from(evt.clipboardData.files || []); const imageFiles = files.filter((file) => file.type.startsWith('image/')); @@ -694,26 +743,6 @@ export default function ChatInput({ const newImages: PastedImage[] = []; for (const file of imageFiles) { - // Check individual file size before processing - if (file.size > MAX_IMAGE_SIZE_MB * 1024 * 1024) { - const errorId = `error-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; - newImages.push({ - id: errorId, - dataUrl: '', - isLoading: false, - error: `Image too large (${Math.round(file.size / (1024 * 1024))}MB). Maximum ${MAX_IMAGE_SIZE_MB}MB allowed.`, - }); - - // Remove the error message after 5 seconds with cleanup tracking - const timeoutId = setTimeout(() => { - setPastedImages((prev) => prev.filter((img) => img.id !== errorId)); - timeoutRefsRef.current.delete(timeoutId); - }, 5000); - timeoutRefsRef.current.add(timeoutId); - - continue; - } - const imageId = `img-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; // Add the image with loading state @@ -728,29 +757,56 @@ export default function ChatInput({ reader.onload = async (e) => { const dataUrl = e.target?.result as string; if (dataUrl) { - // Update the image with the data URL - setPastedImages((prev) => - prev.map((img) => (img.id === imageId ? { ...img, dataUrl, isLoading: true } : img)) - ); - try { - const result = await window.electron.saveDataUrlToTemp(dataUrl, imageId); + // Compress the image + const compressedDataUrl = await compressImageDataUrl(dataUrl); + + // Update the image with the compressed data URL setPastedImages((prev) => - prev.map((img) => - img.id === result.id - ? { ...img, filePath: result.filePath, error: result.error, isLoading: false } - : img - ) + prev.map((img) => (img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: true } : img)) ); - } catch (err) { - console.error('Error saving pasted image:', err); + + // Also save to temp file for backward compatibility + try { + const result = await window.electron.saveDataUrlToTemp(compressedDataUrl, imageId); + setPastedImages((prev) => + prev.map((img) => + img.id === result.id + ? { ...img, filePath: result.filePath, error: result.error, isLoading: false } + : img + ) + ); + } catch (err) { + console.error('Error saving pasted image:', err); + setPastedImages((prev) => + prev.map((img) => + img.id === imageId + ? { ...img, error: 'Failed to save image via Electron.', isLoading: false } + : img + ) + ); + } + } catch (compressionError) { + console.error('Error compressing image:', compressionError); setPastedImages((prev) => prev.map((img) => img.id === imageId - ? { ...img, error: 'Failed to save image via Electron.', isLoading: false } + ? { + ...img, + dataUrl, + isLoading: false, + error: `Image too large and could not be compressed. Try a smaller image.`, + } : img ) ); + + // Remove the error message after 5 seconds + const timeoutId = setTimeout(() => { + setPastedImages((prev) => prev.filter((img) => img.id !== imageId)); + timeoutRefsRef.current.delete(timeoutId); + }, 5000); + timeoutRefsRef.current.add(timeoutId); } } }; @@ -949,9 +1005,28 @@ export default function ChatInput({ const performSubmit = useCallback( (text?: string) => { + // Extract base64 image data from pasted images (for direct image content) + const imageData: import('../types/message').ImageData[] = pastedImages + .filter((img) => img.dataUrl && !img.error && !img.isLoading) + .map((img) => { + // Extract base64 data and mime type from data URL + // Data URL format: data:image/png;base64,iVBORw0KG... + const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], // base64 data + mimeType: matches[1], // mime type + }; + } + return null; + }) + .filter((img): img is import('../types/message').ImageData => img !== null); + + // Also keep file paths for backward compatibility (will be added to text) const validPastedImageFilesPaths = pastedImages .filter((img) => img.filePath && !img.error && !img.isLoading) .map((img) => img.filePath as string); + // Get paths from all dropped files (both parent and local) const droppedFilePaths = allDroppedFiles .filter((file) => !file.error && !file.isLoading) @@ -959,14 +1034,14 @@ export default function ChatInput({ let textToSend = text ?? displayValue.trim(); - // Combine pasted images and dropped files + // Combine pasted image file paths and dropped files (for backward compatibility) const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths]; if (allFilePaths.length > 0) { const pathsString = allFilePaths.join(' '); textToSend = textToSend ? `${textToSend} ${pathsString}` : pathsString; } - if (textToSend) { + if (textToSend || imageData.length > 0) { if (displayValue.trim()) { LocalMessageStorage.addMessage(displayValue); } else if (allFilePaths.length > 0) { @@ -974,7 +1049,12 @@ export default function ChatInput({ } handleSubmit( - new CustomEvent('submit', { detail: { value: textToSend } }) as unknown as React.FormEvent + new CustomEvent('submit', { + detail: { + value: textToSend, + images: imageData.length > 0 ? imageData : undefined + } + }) as unknown as React.FormEvent ); // Auto-resume queue after sending a NON-interruption message (if it was paused due to interruption) @@ -1097,22 +1177,112 @@ export default function ChatInput({ } }; - const handleFileSelect = async () => { + const fileInputRef = React.useRef(null); + + const handleFileSelect = () => { if (isFilePickerOpen) return; + fileInputRef.current?.click(); + }; + + const handleFileInputChange = async (e: React.ChangeEvent) => { + const files = e.target.files; + if (!files || files.length === 0) return; + setIsFilePickerOpen(true); try { - const path = await window.electron.selectFileOrDirectory(); - if (path) { - const isDirectory = !path.includes('.') || path.endsWith('/'); - trackFileAttached(isDirectory ? 'directory' : 'file'); + const file = files[0]; + const isImage = file.type.startsWith('image/'); + + if (isImage) { + trackFileAttached('file'); + + // Check if we're at the image limit + if (pastedImages.length >= MAX_IMAGES_PER_MESSAGE) { + console.warn(`Maximum ${MAX_IMAGES_PER_MESSAGE} images per message`); + return; + } + // Create a unique ID for this image + const uniqueId = `upload-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + // Add a loading placeholder + setPastedImages(prev => [...prev, { + id: uniqueId, + dataUrl: '', + isLoading: true, + error: undefined + }]); + + try { + // Read the file using FileReader + const reader = new FileReader(); + reader.onload = async (evt) => { + const dataUrl = evt.target?.result as string; + if (dataUrl) { + try { + // Compress the image + const compressedDataUrl = await compressImageDataUrl(dataUrl); + + // Save to temp file + const saveResult = await window.electron.saveDataUrlToTemp(compressedDataUrl, uniqueId); + + if (saveResult.error) { + setPastedImages(prev => prev.map(img => + img.id === uniqueId + ? { ...img, isLoading: false, error: `Failed to save: ${saveResult.error}` } + : img + )); + } else { + setPastedImages(prev => prev.map(img => + img.id === uniqueId + ? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined } + : img + )); + } + } catch (compressionError) { + const errorMessage = compressionError instanceof Error + ? compressionError.message + : 'Failed to compress image'; + setPastedImages(prev => prev.map(img => + img.id === uniqueId + ? { ...img, isLoading: false, error: errorMessage } + : img + )); + } + } + }; + reader.onerror = () => { + setPastedImages(prev => prev.map(img => + img.id === uniqueId + ? { ...img, isLoading: false, error: 'Failed to read image file' } + : img + )); + }; + reader.readAsDataURL(file); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Failed to load image'; + setPastedImages(prev => prev.map(img => + img.id === uniqueId + ? { ...img, isLoading: false, error: errorMessage } + : img + )); + } + } else { + // For non-image files, get the path and add to text + trackFileAttached('file'); + const path = window.electron.getPathForFile(file); const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path; setDisplayValue(newValue); setValue(newValue); - textAreaRef.current?.focus(); } + + textAreaRef.current?.focus(); } finally { setIsFilePickerOpen(false); + // Reset the input so the same file can be selected again + if (e.target) { + e.target.value = ''; + } } }; @@ -1244,6 +1414,14 @@ export default function ChatInput({ onDrop={handleLocalDrop} onDragOver={handleLocalDragOver} > + {/* Hidden file input */} + {/* Message Queue Display */} {queuedMessages.length > 0 && ( - Attach file or directory + Attach file
{/* Model selector, mode selector, alerts, summarize button */} diff --git a/ui/desktop/src/hooks/useChatStream.ts b/ui/desktop/src/hooks/useChatStream.ts index c4acd595bc7..3e219a5af11 100644 --- a/ui/desktop/src/hooks/useChatStream.ts +++ b/ui/desktop/src/hooks/useChatStream.ts @@ -38,7 +38,7 @@ interface UseChatStreamReturn { messages: Message[]; chatState: ChatState; setChatState: (state: ChatState) => void; - handleSubmit: (userMessage: string) => Promise; + handleSubmit: (userMessage: string, images?: import('../types/message').ImageData[]) => Promise; submitElicitationResponse: ( elicitationId: string, userData: Record @@ -446,7 +446,7 @@ export function useChatStream({ }, [sessionId, onSessionLoaded]); const handleSubmit = useCallback( - async (userMessage: string) => { + async (userMessage: string, images?: import('../types/message').ImageData[]) => { const currentState = stateRef.current; // Guard: Don't submit if session hasn't been loaded yet @@ -455,7 +455,7 @@ export function useChatStream({ } const hasExistingMessages = currentState.messages.length > 0; - const hasNewMessage = userMessage.trim().length > 0; + const hasNewMessage = userMessage.trim().length > 0 || (images && images.length > 0); // Don't submit if there's no message and no conversation to continue if (!hasNewMessage && !hasExistingMessages) { @@ -517,7 +517,7 @@ export function useChatStream({ } const newMessage = hasNewMessage - ? createUserMessage(userMessage) + ? createUserMessage(userMessage, images) : currentState.messages[currentState.messages.length - 1]; const currentMessages = hasNewMessage ? [...currentState.messages, newMessage] diff --git a/ui/desktop/src/types/message.ts b/ui/desktop/src/types/message.ts index ba4425fddea..7e54514c3f2 100644 --- a/ui/desktop/src/types/message.ts +++ b/ui/desktop/src/types/message.ts @@ -7,12 +7,35 @@ export type NotificationEvent = Extract; // Compaction response message - must match backend constant const COMPACTION_THINKING_TEXT = 'goose is compacting the conversation...'; -export function createUserMessage(text: string): Message { +export interface ImageData { + data: string; // base64 encoded image data + mimeType: string; +} + +export function createUserMessage(text: string, images?: ImageData[]): Message { + const content: Message['content'] = []; + + // Add text content if present + if (text.trim()) { + content.push({ type: 'text', text }); + } + + // Add image content if present + if (images && images.length > 0) { + images.forEach(img => { + content.push({ + type: 'image', + data: img.data, + mimeType: img.mimeType, + }); + }); + } + return { id: generateMessageId(), role: 'user', created: Math.floor(Date.now() / 1000), - content: [{ type: 'text', text }], + content, metadata: { userVisible: true, agentVisible: true }, }; } From 7d7bccf05905ea8df036f7a15182468e98a58df6 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Wed, 21 Jan 2026 17:03:33 -0500 Subject: [PATCH 02/13] WIP --- ui/desktop/package-lock.json | 40 +++------- ui/desktop/src/components/BaseChat.tsx | 7 +- ui/desktop/src/components/ChatInput.tsx | 101 ++++++------------------ ui/desktop/src/hooks/useChatStream.ts | 3 +- 4 files changed, 40 insertions(+), 111 deletions(-) diff --git a/ui/desktop/package-lock.json b/ui/desktop/package-lock.json index e8957a3c22d..82f97608718 100644 --- a/ui/desktop/package-lock.json +++ b/ui/desktop/package-lock.json @@ -217,7 +217,6 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -587,7 +586,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -631,7 +629,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -1089,7 +1086,6 @@ "integrity": "sha512-zx0EIq78WlY/lBb1uXlziZmDZI4ubcCXIMJ4uGjXzZW0nS19TjSPeXPAjzzTmKQlJUZm0SbmZhPKP7tuQ1SsEw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "chalk": "^4.1.1", "fs-extra": "^9.0.1", @@ -2659,7 +2655,6 @@ "integrity": "sha512-yl43JD/86CIj3Mz5mvvLJqAOfIup7ncxfJ0Btnl0/v5TouVUyeEdcpknfgc+yMevS/48oH9WAkkw93m7otLb/A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@inquirer/checkbox": "^3.0.1", "@inquirer/confirm": "^4.0.1", @@ -6226,7 +6221,8 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -6514,7 +6510,6 @@ "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6556,7 +6551,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.8.tgz", "integrity": "sha512-3MbSL37jEchWZz2p2mjntRZtPt837ij10ApxKfgmXCTuHWagYg7iA5bqPw6C8BMPfwidlvfPI/fxOc42HLhcyg==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -6567,7 +6561,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -6708,7 +6701,6 @@ "integrity": "sha512-npiaib8XzbjtzS2N4HlqPvlpxpmZ14FjSJrteZpPxGUaYPlvhzlzUZ4mZyABo0EFrOWnvyd0Xxroq//hKhtAWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.53.0", "@typescript-eslint/types": "8.53.0", @@ -7094,7 +7086,6 @@ "integrity": "sha512-hRDjg6dlDz7JlZAvjbiCdAJ3SDG+NH8tjZe21vjxfvT2ssYAn72SRXMge3dKKABm3bIJ3C+3wdunIdur8PHEAw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/utils": "4.0.17", "fflate": "^0.8.2", @@ -7343,7 +7334,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -7969,7 +7959,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -9268,7 +9257,8 @@ "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/dom-helpers": { "version": "5.2.1", @@ -9326,7 +9316,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@electron/get": "^2.0.0", "@types/node": "^22.7.7", @@ -10341,7 +10330,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -10801,7 +10789,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -13086,7 +13073,6 @@ "integrity": "sha512-mjzqwWRD9Y1J1KUi7W97Gja1bwOOM5Ug0EZ6UDK3xS7j7mndrkwozHtSblfomlzyB4NepioNt+B2sOSzczVgtQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@acemir/cssom": "^0.9.28", "@asamuzakjp/dom-selector": "^6.7.6", @@ -14330,6 +14316,7 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -14350,7 +14337,6 @@ "integrity": "sha512-xrHS24IxaLrvuo613F719wvOIv9xPHFWQHuvGUBmPnCA/3MQxKI3b+r7n1jAoDHmsbC5bRhTZYR77invLAxVnw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/parser": "^7.28.5", "@babel/types": "^7.28.5", @@ -16721,7 +16707,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -16823,6 +16808,7 @@ "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -16838,6 +16824,7 @@ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -17193,7 +17180,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -17203,7 +17189,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -17225,7 +17210,8 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/react-markdown": { "version": "10.1.0", @@ -18209,7 +18195,6 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -19142,8 +19127,7 @@ "version": "4.1.18", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.18.tgz", "integrity": "sha512-4+Z+0yiYyEtUVCScyfHCxOYP06L5Ne+JiHhY2IjR2KWMIWhJOYZKLSGZaP5HkZ8+bY0cxfzwDE5uOmzFXyIwxw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/tailwindcss-animate": { "version": "1.0.7", @@ -19673,7 +19657,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -20088,7 +20071,6 @@ "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -20179,7 +20161,6 @@ "integrity": "sha512-FQMeF0DJdWY0iOnbv466n/0BudNdKj1l5jYgl5JVTwjSsZSlqyXFt/9+1sEyhR6CLowbZpV7O1sCHrzBhucKKg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/expect": "4.0.17", "@vitest/mocker": "4.0.17", @@ -20839,7 +20820,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 4f688a234f9..96954ae87c0 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -28,6 +28,7 @@ import { useNavigation } from '../hooks/useNavigation'; import { RecipeHeader } from './RecipeHeader'; import { RecipeWarningModal } from './ui/RecipeWarningModal'; import { scanRecipe } from '../recipe'; +import { ImageData } from '../types/message'; import { useCostTracking } from '../hooks/useCostTracking'; import RecipeActivities from './recipes/RecipeActivities'; import { useToolCount } from './alerts/useToolCount'; @@ -159,10 +160,10 @@ function BaseChatContent({ .reverse(); }, [messages]); - const handleFormSubmit = (e: React.FormEvent) => { + const chatInputSubmit = (e: React.FormEvent) => { const customEvent = e as unknown as CustomEvent; const textValue = customEvent.detail?.value || ''; - const images = customEvent.detail?.images as import('../types/message').ImageData[] | undefined; + const images = customEvent.detail?.images as ImageData[] | undefined; if (recipe && textValue.trim()) { setHasStartedUsingRecipe(true); @@ -458,7 +459,7 @@ function BaseChatContent({ void; + handleSubmit: (userMessage: string, images: ImageData[]) => void; chatState: ChatState; setChatState?: (state: ChatState) => void; onStop?: () => void; @@ -220,9 +221,7 @@ export default function ChatInput({ const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); handleSubmit( - new CustomEvent('submit', { - detail: { value: nextMessage.content }, - }) as unknown as React.FormEvent + nextMessage.content, nextMessage.images ); setQueuedMessages((prev) => { const newQueue = prev.slice(1); @@ -512,12 +511,7 @@ export default function ChatInput({ compactButtonDisabled: !totalTokens, onCompact: () => { window.dispatchEvent(new CustomEvent(AppEvents.HIDE_ALERT_POPOVER)); - - const customEvent = new CustomEvent('submit', { - detail: { value: MANUAL_COMPACT_TRIGGER }, - }) as unknown as React.FormEvent; - - handleSubmit(customEvent); + handleSubmit(MANUAL_COMPACT_TRIGGER, []); }, compactIcon: , }); @@ -695,13 +689,7 @@ export default function ChatInput({ // Convert to JPEG with 0.85 quality const compressedDataUrl = canvas.toDataURL('image/jpeg', 0.85); - const compressedSizeMB = (compressedDataUrl.length * 3) / 4 / (1024 * 1024); - - if (compressedSizeMB > MAX_IMAGE_SIZE_MB) { - reject(new Error(`Image is ${Math.round(compressedSizeMB)}MB after compression, exceeds ${MAX_IMAGE_SIZE_MB}MB limit`)); - } else { - resolve(compressedDataUrl); - } + resolve(compressedDataUrl); }; img.onerror = () => reject(new Error('Failed to load image for compression')); img.src = dataUrl; @@ -763,29 +751,8 @@ export default function ChatInput({ // Update the image with the compressed data URL setPastedImages((prev) => - prev.map((img) => (img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: true } : img)) + prev.map((img) => (img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: false } : img)) ); - - // Also save to temp file for backward compatibility - try { - const result = await window.electron.saveDataUrlToTemp(compressedDataUrl, imageId); - setPastedImages((prev) => - prev.map((img) => - img.id === result.id - ? { ...img, filePath: result.filePath, error: result.error, isLoading: false } - : img - ) - ); - } catch (err) { - console.error('Error saving pasted image:', err); - setPastedImages((prev) => - prev.map((img) => - img.id === imageId - ? { ...img, error: 'Failed to save image via Electron.', isLoading: false } - : img - ) - ); - } } catch (compressionError) { console.error('Error compressing image:', compressionError); setPastedImages((prev) => @@ -927,17 +894,13 @@ export default function ChatInput({ return false; } - const validPastedImageFilesPaths = pastedImages - .filter((img) => img.filePath && !img.error && !img.isLoading) - .map((img) => img.filePath as string); const droppedFilePaths = allDroppedFiles .filter((file) => !file.error && !file.isLoading) .map((file) => file.path); let contentToQueue = displayValue.trim(); - const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths]; - if (allFilePaths.length > 0) { - const pathsString = allFilePaths.join(' '); + if (droppedFilePaths.length > 0) { + const pathsString = droppedFilePaths.join(' '); contentToQueue = contentToQueue ? `${contentToQueue} ${pathsString}` : pathsString; } @@ -1000,7 +963,7 @@ export default function ChatInput({ const canSubmit = !isLoading && (displayValue.trim() || - pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) || + pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || allDroppedFiles.some((file) => !file.error && !file.isLoading)); const performSubmit = useCallback( @@ -1022,30 +985,24 @@ export default function ChatInput({ }) .filter((img): img is import('../types/message').ImageData => img !== null); - // Also keep file paths for backward compatibility (will be added to text) - const validPastedImageFilesPaths = pastedImages - .filter((img) => img.filePath && !img.error && !img.isLoading) - .map((img) => img.filePath as string); - - // Get paths from all dropped files (both parent and local) + // Get paths from all dropped files const droppedFilePaths = allDroppedFiles .filter((file) => !file.error && !file.isLoading) .map((file) => file.path); let textToSend = text ?? displayValue.trim(); - // Combine pasted image file paths and dropped files (for backward compatibility) - const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths]; - if (allFilePaths.length > 0) { - const pathsString = allFilePaths.join(' '); + // Add dropped file paths to text + if (droppedFilePaths.length > 0) { + const pathsString = droppedFilePaths.join(' '); textToSend = textToSend ? `${textToSend} ${pathsString}` : pathsString; } if (textToSend || imageData.length > 0) { if (displayValue.trim()) { LocalMessageStorage.addMessage(displayValue); - } else if (allFilePaths.length > 0) { - LocalMessageStorage.addMessage(allFilePaths.join(' ')); + } else if (droppedFilePaths.length > 0) { + LocalMessageStorage.addMessage(droppedFilePaths.join(' ')); } handleSubmit( @@ -1170,7 +1127,7 @@ export default function ChatInput({ const canSubmit = !isLoading && (displayValue.trim() || - pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) || + pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || allDroppedFiles.some((file) => !file.error && !file.isLoading)); if (canSubmit) { performSubmit(); @@ -1223,22 +1180,12 @@ export default function ChatInput({ // Compress the image const compressedDataUrl = await compressImageDataUrl(dataUrl); - // Save to temp file - const saveResult = await window.electron.saveDataUrlToTemp(compressedDataUrl, uniqueId); - - if (saveResult.error) { - setPastedImages(prev => prev.map(img => - img.id === uniqueId - ? { ...img, isLoading: false, error: `Failed to save: ${saveResult.error}` } - : img - )); - } else { - setPastedImages(prev => prev.map(img => - img.id === uniqueId - ? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined } - : img - )); - } + // Update the image with the compressed data URL + setPastedImages(prev => prev.map(img => + img.id === uniqueId + ? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined } + : img + )); } catch (compressionError) { const errorMessage = compressionError instanceof Error ? compressionError.message @@ -1310,7 +1257,7 @@ export default function ChatInput({ const hasSubmittableContent = displayValue.trim() || - pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) || + pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || allDroppedFiles.some((file) => !file.error && !file.isLoading); const isAnyImageLoading = pastedImages.some((img) => img.isLoading); const isAnyDroppedFileLoading = allDroppedFiles.some((file) => file.isLoading); diff --git a/ui/desktop/src/hooks/useChatStream.ts b/ui/desktop/src/hooks/useChatStream.ts index 3e219a5af11..5cb406200d8 100644 --- a/ui/desktop/src/hooks/useChatStream.ts +++ b/ui/desktop/src/hooks/useChatStream.ts @@ -21,6 +21,7 @@ import { getCompactingMessage, getThinkingMessage, NotificationEvent, + ImageData, } from '../types/message'; import { errorMessage } from '../utils/conversionUtils'; import { showExtensionLoadResults } from '../utils/extensionErrorUtils'; @@ -38,7 +39,7 @@ interface UseChatStreamReturn { messages: Message[]; chatState: ChatState; setChatState: (state: ChatState) => void; - handleSubmit: (userMessage: string, images?: import('../types/message').ImageData[]) => Promise; + handleSubmit: (userMessage: string, images: ImageData[]) => Promise; submitElicitationResponse: ( elicitationId: string, userData: Record From 008e710bcc8b0929b17aed0c19d653845a78f641 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Wed, 21 Jan 2026 17:31:40 -0500 Subject: [PATCH 03/13] check point --- ui/desktop/src/components/BaseChat.tsx | 12 +++---- ui/desktop/src/components/ChatInput.tsx | 41 +++++----------------- ui/desktop/src/components/Hub.tsx | 14 +++----- ui/desktop/src/components/MessageQueue.tsx | 4 ++- ui/desktop/src/hooks/useAutoSubmit.ts | 9 ++--- ui/desktop/src/hooks/useChatStream.ts | 6 ++-- 6 files changed, 29 insertions(+), 57 deletions(-) diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 96954ae87c0..8ba974e3f78 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -160,11 +160,7 @@ function BaseChatContent({ .reverse(); }, [messages]); - const chatInputSubmit = (e: React.FormEvent) => { - const customEvent = e as unknown as CustomEvent; - const textValue = customEvent.detail?.value || ''; - const images = customEvent.detail?.images as ImageData[] | undefined; - + const chatInputSubmit = (textValue: string, images: ImageData[]) => { if (recipe && textValue.trim()) { setHasStartedUsingRecipe(true); } @@ -408,7 +404,7 @@ function BaseChatContent({ {recipe && (
handleSubmit(text)} + append={(text: string) => handleSubmit(text, [])} activities={Array.isArray(recipe.activities) ? recipe.activities : null} title={recipe.title} parameterValues={session?.user_recipe_values || {}} @@ -423,7 +419,7 @@ function BaseChatContent({ messages={messages} chat={{ sessionId }} toolCallNotifications={toolCallNotifications} - append={(text: string) => handleSubmit(text)} + append={(text: string) => handleSubmit(text, [])} isUserMessage={(m: Message) => m.role === 'user'} isStreamingMessage={chatState !== ChatState.Idle} onRenderingComplete={handleRenderingComplete} @@ -435,7 +431,7 @@ function BaseChatContent({
) : !recipe && showPopularTopics ? ( - handleSubmit(text)} /> + handleSubmit(text, [])} /> ) : null} diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index 1c1677474a6..bb67de5419a 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -25,7 +25,7 @@ import { COST_TRACKING_ENABLED, VOICE_DICTATION_ELEVENLABS_ENABLED } from '../up import { CostTracker } from './bottom_menu/CostTracker'; import { DroppedFile, useFileDrop } from '../hooks/useFileDrop'; import { Recipe } from '../recipe'; -import MessageQueue from './MessageQueue'; +import {MessageQueue, QueuedMessage} from './MessageQueue'; import { detectInterruption } from '../utils/interruptionDetector'; import { DiagnosticsModal } from './ui/Diagnostics'; import { getSession, Message } from '../api'; @@ -43,13 +43,6 @@ import { import { getNavigationShortcutText } from '../utils/keyboardShortcuts'; import { ImageData } from '../types/message'; -interface QueuedMessage { - id: string; - content: string; - timestamp: number; - images: ImageData[]; -} - interface PastedImage { id: string; dataUrl: string; // For immediate preview @@ -653,17 +646,14 @@ export default function ChatInput({ })); }; - // Helper function to compress/resize image - // Always resize to max 1024px on longest side and compress as JPEG at 0.85 quality const compressImageDataUrl = async (dataUrl: string): Promise => { return new Promise((resolve, reject) => { - const img = new Image(); + const img = new globalThis.Image(); img.onload = () => { const maxDimension = 1024; let width = img.width; let height = img.height; - // Calculate new dimensions maintaining aspect ratio if (width > maxDimension || height > maxDimension) { if (width > height) { height = Math.floor((height * maxDimension) / width); @@ -913,10 +903,11 @@ export default function ChatInput({ // For interruptions, we need to queue the message to be sent after the stop completes // rather than trying to send it immediately while the system is still loading - const interruptionMessage = { + const interruptionMessage: QueuedMessage = { id: Date.now().toString() + Math.random().toString(36).substr(2, 9), content: contentToQueue, timestamp: Date.now(), + images: [], }; // Add the interruption message to the front of the queue so it gets sent first @@ -934,10 +925,11 @@ export default function ChatInput({ return true; } - const newMessage = { + const newMessage: QueuedMessage = { id: Date.now().toString() + Math.random().toString(36).substr(2, 9), content: contentToQueue, timestamp: Date.now(), + images: [], }; setQueuedMessages((prev) => { const newQueue = [...prev, newMessage]; @@ -1005,14 +997,7 @@ export default function ChatInput({ LocalMessageStorage.addMessage(droppedFilePaths.join(' ')); } - handleSubmit( - new CustomEvent('submit', { - detail: { - value: textToSend, - images: imageData.length > 0 ? imageData : undefined - } - }) as unknown as React.FormEvent - ); + handleSubmit(textToSend, imageData); // Auto-resume queue after sending a NON-interruption message (if it was paused due to interruption) if ( @@ -1313,11 +1298,7 @@ export default function ChatInput({ // Remove the message from queue and send it immediately setQueuedMessages((prev) => prev.filter((msg) => msg.id !== messageId)); LocalMessageStorage.addMessage(messageToSend.content); - handleSubmit( - new CustomEvent('submit', { - detail: { value: messageToSend.content }, - }) as unknown as React.FormEvent - ); + handleSubmit(messageToSend.content, messageToSend.images); // Restore previous pause state after a brief delay to prevent race condition setTimeout(() => { @@ -1331,11 +1312,7 @@ export default function ChatInput({ if (!isLoading && queuedMessages.length > 0) { const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); - handleSubmit( - new CustomEvent('submit', { - detail: { value: nextMessage.content }, - }) as unknown as React.FormEvent - ); + handleSubmit(nextMessage.content, nextMessage.images); setQueuedMessages((prev) => { const newQueue = prev.slice(1); // If queue becomes empty after processing, clear the paused state diff --git a/ui/desktop/src/components/Hub.tsx b/ui/desktop/src/components/Hub.tsx index cdd62f6e04d..814cdb23d93 100644 --- a/ui/desktop/src/components/Hub.tsx +++ b/ui/desktop/src/components/Hub.tsx @@ -29,6 +29,7 @@ import { import { getInitialWorkingDir } from '../utils/workingDir'; import { createSession } from '../sessions'; import LoadingGoose from './LoadingGoose'; +import { ImageData } from '../types/message'; export default function Hub({ setView, @@ -39,11 +40,8 @@ export default function Hub({ const [workingDir, setWorkingDir] = useState(getInitialWorkingDir()); const [isCreatingSession, setIsCreatingSession] = useState(false); - const handleSubmit = async (e: React.FormEvent) => { - const customEvent = e as unknown as CustomEvent; - const combinedTextFromInput = customEvent.detail?.value || ''; - - if (combinedTextFromInput.trim() && !isCreatingSession) { + const handleSubmit = async (userMessage: string, _images: ImageData[]) => { + if (userMessage.trim() && !isCreatingSession) { const extensionConfigs = getExtensionConfigsWithOverrides(extensionsList); clearExtensionOverrides(); setIsCreatingSession(true); @@ -57,21 +55,19 @@ export default function Hub({ window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED)); window.dispatchEvent( new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, { - detail: { sessionId: session.id, initialMessage: combinedTextFromInput }, + detail: { sessionId: session.id, initialMessage: userMessage }, }) ); setView('pair', { disableAnimation: true, resumeSessionId: session.id, - initialMessage: combinedTextFromInput, + initialMessage: userMessage, }); } catch (error) { console.error('Failed to create session:', error); setIsCreatingSession(false); } - - e.preventDefault(); } }; diff --git a/ui/desktop/src/components/MessageQueue.tsx b/ui/desktop/src/components/MessageQueue.tsx index a580939086c..2eb783ee098 100644 --- a/ui/desktop/src/components/MessageQueue.tsx +++ b/ui/desktop/src/components/MessageQueue.tsx @@ -1,11 +1,13 @@ import React, { useState } from 'react'; import { X, Clock, Send, GripVertical, Zap, Sparkles, ChevronDown, ChevronUp } from 'lucide-react'; import { Button } from './ui/button'; +import { ImageData } from '../types/message'; -interface QueuedMessage { +export interface QueuedMessage { id: string; content: string; timestamp: number; + images: ImageData[]; } interface MessageQueueProps { diff --git a/ui/desktop/src/hooks/useAutoSubmit.ts b/ui/desktop/src/hooks/useAutoSubmit.ts index 05e6df9664c..b78bb5cc615 100644 --- a/ui/desktop/src/hooks/useAutoSubmit.ts +++ b/ui/desktop/src/hooks/useAutoSubmit.ts @@ -4,6 +4,7 @@ import { useSearchParams } from 'react-router-dom'; import { Session } from '../api'; import { Message } from '../api'; import { ChatState } from '../types/chatState'; +import { ImageData } from '../types/message'; /** * Auto-submit scenarios: @@ -18,7 +19,7 @@ interface UseAutoSubmitProps { messages: Message[]; chatState: ChatState; initialMessage: string | undefined; - handleSubmit: (message: string) => void; + handleSubmit: (message: string, images: ImageData[]) => void; } interface UseAutoSubmitReturn { @@ -68,7 +69,7 @@ export function useAutoSubmit({ // Hub always creates new sessions, so message_count will be 0 if (initialMessage && session.message_count === 0 && messages.length === 0) { hasAutoSubmittedRef.current = true; - handleSubmit(initialMessage); + handleSubmit(initialMessage, []); clearInitialMessage(); return; } @@ -76,7 +77,7 @@ export function useAutoSubmit({ // Scenario 2: Forked session with edited message if (shouldStartAgent && initialMessage) { hasAutoSubmittedRef.current = true; - handleSubmit(initialMessage); + handleSubmit(initialMessage, []); clearInitialMessage(); return; } @@ -84,7 +85,7 @@ export function useAutoSubmit({ // Scenario 3: Resume with shouldStartAgent (continue existing conversation) if (shouldStartAgent) { hasAutoSubmittedRef.current = true; - handleSubmit(''); + handleSubmit('', []); } }, [ session, diff --git a/ui/desktop/src/hooks/useChatStream.ts b/ui/desktop/src/hooks/useChatStream.ts index 5cb406200d8..0e0a10c7a75 100644 --- a/ui/desktop/src/hooks/useChatStream.ts +++ b/ui/desktop/src/hooks/useChatStream.ts @@ -447,7 +447,7 @@ export function useChatStream({ }, [sessionId, onSessionLoaded]); const handleSubmit = useCallback( - async (userMessage: string, images?: import('../types/message').ImageData[]) => { + async (userMessage: string, images: ImageData[]) => { const currentState = stateRef.current; // Guard: Don't submit if session hasn't been loaded yet @@ -456,7 +456,7 @@ export function useChatStream({ } const hasExistingMessages = currentState.messages.length > 0; - const hasNewMessage = userMessage.trim().length > 0 || (images && images.length > 0); + const hasNewMessage = userMessage.trim().length > 0 || images.length > 0; // Don't submit if there's no message and no conversation to continue if (!hasNewMessage && !hasExistingMessages) { @@ -694,7 +694,7 @@ export function useChatStream({ if (sessionResponse.data?.conversation) { dispatch({ type: 'SET_MESSAGES', payload: sessionResponse.data.conversation }); } - await handleSubmit(newContent); + await handleSubmit(newContent, []); } } catch (error) { const errorMsg = errorMessage(error); From 3dcfe10324bfffcd56e392a4667811b8784c1696 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Wed, 21 Jan 2026 20:22:34 -0500 Subject: [PATCH 04/13] well well well --- ui/desktop/package-lock.json | 40 ++++++--- ui/desktop/src/App.tsx | 11 +-- ui/desktop/src/components/BaseChat.tsx | 15 ++-- .../src/components/ChatSessionsContainer.tsx | 3 +- ui/desktop/src/components/GooseMessage.tsx | 16 ++-- .../components/GooseSidebar/AppSidebar.tsx | 2 +- ui/desktop/src/components/Hub.tsx | 8 +- ui/desktop/src/components/ImagePreview.tsx | 86 ++++--------------- .../src/components/Layout/AppLayout.tsx | 5 +- ui/desktop/src/components/UserMessage.tsx | 45 ++++------ .../sessions/SessionViewComponents.tsx | 18 +--- ui/desktop/src/hooks/useAutoSubmit.ts | 6 +- ui/desktop/src/main.ts | 73 ---------------- ui/desktop/src/preload.ts | 5 -- ui/desktop/src/sessions.ts | 6 +- ui/desktop/src/types/message.ts | 23 +++-- ui/desktop/src/utils/imageUtils.ts | 59 ------------- ui/desktop/src/utils/navigationUtils.ts | 5 +- ui/desktop/src/utils/toolCallChaining.ts | 17 +--- 19 files changed, 121 insertions(+), 322 deletions(-) delete mode 100644 ui/desktop/src/utils/imageUtils.ts diff --git a/ui/desktop/package-lock.json b/ui/desktop/package-lock.json index 82f97608718..e8957a3c22d 100644 --- a/ui/desktop/package-lock.json +++ b/ui/desktop/package-lock.json @@ -217,6 +217,7 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -586,6 +587,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -629,6 +631,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -1086,6 +1089,7 @@ "integrity": "sha512-zx0EIq78WlY/lBb1uXlziZmDZI4ubcCXIMJ4uGjXzZW0nS19TjSPeXPAjzzTmKQlJUZm0SbmZhPKP7tuQ1SsEw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "chalk": "^4.1.1", "fs-extra": "^9.0.1", @@ -2655,6 +2659,7 @@ "integrity": "sha512-yl43JD/86CIj3Mz5mvvLJqAOfIup7ncxfJ0Btnl0/v5TouVUyeEdcpknfgc+yMevS/48oH9WAkkw93m7otLb/A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@inquirer/checkbox": "^3.0.1", "@inquirer/confirm": "^4.0.1", @@ -6221,8 +6226,7 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -6510,6 +6514,7 @@ "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6551,6 +6556,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.8.tgz", "integrity": "sha512-3MbSL37jEchWZz2p2mjntRZtPt837ij10ApxKfgmXCTuHWagYg7iA5bqPw6C8BMPfwidlvfPI/fxOc42HLhcyg==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -6561,6 +6567,7 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -6701,6 +6708,7 @@ "integrity": "sha512-npiaib8XzbjtzS2N4HlqPvlpxpmZ14FjSJrteZpPxGUaYPlvhzlzUZ4mZyABo0EFrOWnvyd0Xxroq//hKhtAWg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.53.0", "@typescript-eslint/types": "8.53.0", @@ -7086,6 +7094,7 @@ "integrity": "sha512-hRDjg6dlDz7JlZAvjbiCdAJ3SDG+NH8tjZe21vjxfvT2ssYAn72SRXMge3dKKABm3bIJ3C+3wdunIdur8PHEAw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/utils": "4.0.17", "fflate": "^0.8.2", @@ -7334,6 +7343,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -7959,6 +7969,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -9257,8 +9268,7 @@ "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/dom-helpers": { "version": "5.2.1", @@ -9316,6 +9326,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@electron/get": "^2.0.0", "@types/node": "^22.7.7", @@ -10330,6 +10341,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -10789,6 +10801,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -13073,6 +13086,7 @@ "integrity": "sha512-mjzqwWRD9Y1J1KUi7W97Gja1bwOOM5Ug0EZ6UDK3xS7j7mndrkwozHtSblfomlzyB4NepioNt+B2sOSzczVgtQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@acemir/cssom": "^0.9.28", "@asamuzakjp/dom-selector": "^6.7.6", @@ -14316,7 +14330,6 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -14337,6 +14350,7 @@ "integrity": "sha512-xrHS24IxaLrvuo613F719wvOIv9xPHFWQHuvGUBmPnCA/3MQxKI3b+r7n1jAoDHmsbC5bRhTZYR77invLAxVnw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/parser": "^7.28.5", "@babel/types": "^7.28.5", @@ -16707,6 +16721,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -16808,7 +16823,6 @@ "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -16824,7 +16838,6 @@ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -17180,6 +17193,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -17189,6 +17203,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -17210,8 +17225,7 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/react-markdown": { "version": "10.1.0", @@ -18195,6 +18209,7 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -19127,7 +19142,8 @@ "version": "4.1.18", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.18.tgz", "integrity": "sha512-4+Z+0yiYyEtUVCScyfHCxOYP06L5Ne+JiHhY2IjR2KWMIWhJOYZKLSGZaP5HkZ8+bY0cxfzwDE5uOmzFXyIwxw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/tailwindcss-animate": { "version": "1.0.7", @@ -19657,6 +19673,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -20071,6 +20088,7 @@ "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -20161,6 +20179,7 @@ "integrity": "sha512-FQMeF0DJdWY0iOnbv466n/0BudNdKj1l5jYgl5JVTwjSsZSlqyXFt/9+1sEyhR6CLowbZpV7O1sCHrzBhucKKg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.0.17", "@vitest/mocker": "4.0.17", @@ -20820,6 +20839,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/ui/desktop/src/App.tsx b/ui/desktop/src/App.tsx index 5c1f50f31c0..2031ee7ccd5 100644 --- a/ui/desktop/src/App.tsx +++ b/ui/desktop/src/App.tsx @@ -20,10 +20,11 @@ import { createSession } from './sessions'; import { ChatType } from './types/chat'; import Hub from './components/Hub'; +import { ImageData } from './types/message'; interface PairRouteState { resumeSessionId?: string; - initialMessage?: string; + initialMessage?: { msg: string; images: ImageData[] }; } import SettingsView, { SettingsViewOptions } from './components/settings/SettingsView'; import SessionsView from './components/sessions/SessionsView'; @@ -67,8 +68,8 @@ const HubRouteWrapper = () => { const PairRouteWrapper = ({ activeSessions, }: { - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; - setActiveSessions: (sessions: Array<{ sessionId: string; initialMessage?: string }>) => void; + activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; + setActiveSessions: (sessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>) => void; }) => { const { extensionsList } = useConfig(); const location = useLocation(); @@ -353,13 +354,13 @@ export function AppInner() { const MAX_ACTIVE_SESSIONS = 10; const [activeSessions, setActiveSessions] = useState< - Array<{ sessionId: string; initialMessage?: string }> + Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }> >([]); useEffect(() => { const handleAddActiveSession = (event: Event) => { const { sessionId, initialMessage } = ( - event as CustomEvent<{ sessionId: string; initialMessage?: string }> + event as CustomEvent<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }> ).detail; setActiveSessions((prev) => { diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 8ba974e3f78..9e66db67ba7 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -32,14 +32,14 @@ import { ImageData } from '../types/message'; import { useCostTracking } from '../hooks/useCostTracking'; import RecipeActivities from './recipes/RecipeActivities'; import { useToolCount } from './alerts/useToolCount'; -import { getThinkingMessage, getTextContent } from '../types/message'; +import { getThinkingMessage, getTextAndImageContent } from '../types/message'; import ParameterInputModal from './ParameterInputModal'; import { substituteParameters } from '../utils/providerUtils'; import CreateRecipeFromSessionModal from './recipes/CreateRecipeFromSessionModal'; import { toastSuccess } from '../toasts'; import { Recipe } from '../recipe'; import { useAutoSubmit } from '../hooks/useAutoSubmit'; -import { Goose } from './icons/Goose'; +import { Goose } from './icons'; import EnvironmentBadge from './GooseSidebar/EnvironmentBadge'; const CurrentModelContext = createContext<{ model: string; mode: string } | null>(null); @@ -57,10 +57,10 @@ interface BaseChatProps { suppressEmptyState: boolean; sessionId: string; isActiveSession: boolean; - initialMessage?: string; + initialMessage?: { msg: string; images: ImageData[] }; } -function BaseChatContent({ +export default function BaseChat({ setChat, renderHeader, customChatInputProps = {}, @@ -150,7 +150,7 @@ function BaseChatContent({ return messages .reduce((history, message) => { if (message.role === 'user') { - const text = getTextContent(message).trim(); + const text = getTextAndImageContent(message).textContent.trim(); if (text) { history.push(text); } @@ -324,7 +324,6 @@ function BaseChatContent({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [session?.name, setChat]); - // Only use initialMessage for the prompt if it hasn't been submitted yet // If we have a recipe prompt and user recipe values, substitute parameters let recipePrompt = ''; if (messages.length === 0 && recipe?.prompt) { @@ -518,7 +517,3 @@ function BaseChatContent({
); } - -export default function BaseChat(props: BaseChatProps) { - return ; -} diff --git a/ui/desktop/src/components/ChatSessionsContainer.tsx b/ui/desktop/src/components/ChatSessionsContainer.tsx index 3afac2ce8de..0047bd7f06a 100644 --- a/ui/desktop/src/components/ChatSessionsContainer.tsx +++ b/ui/desktop/src/components/ChatSessionsContainer.tsx @@ -1,10 +1,11 @@ import { useSearchParams } from 'react-router-dom'; import BaseChat from './BaseChat'; import { ChatType } from '../types/chat'; +import { ImageData } from '../types/message'; interface ChatSessionsContainerProps { setChat: (chat: ChatType) => void; - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; + activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; } /** diff --git a/ui/desktop/src/components/GooseMessage.tsx b/ui/desktop/src/components/GooseMessage.tsx index cbcf47fee23..f916030e8a2 100644 --- a/ui/desktop/src/components/GooseMessage.tsx +++ b/ui/desktop/src/components/GooseMessage.tsx @@ -1,11 +1,10 @@ import { useMemo, useRef } from 'react'; import ImagePreview from './ImagePreview'; -import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils'; import { formatMessageTimestamp } from '../utils/timeUtils'; import MarkdownContent from './MarkdownContent'; import ToolCallWithResponse from './ToolCallWithResponse'; import { - getTextContent, + getTextAndImageContent, getToolRequests, getToolResponses, getToolConfirmationContent, @@ -45,28 +44,25 @@ export default function GooseMessage({ }: GooseMessageProps) { const contentRef = useRef(null); - let textContent = getTextContent(message); + let {textContent, imagePaths} = getTextAndImageContent(message); - const splitChainOfThought = (text: string): { visibleText: string; cotText: string | null } => { + const splitChainOfThought = (text: string): { displayText: string; cotText: string | null } => { const regex = /([\s\S]*?)<\/think>/i; const match = text.match(regex); if (!match) { - return { visibleText: text, cotText: null }; + return { displayText: text, cotText: null }; } const cotRaw = match[1].trim(); const visibleText = text.replace(regex, '').trim(); return { - visibleText, + displayText: visibleText, cotText: cotRaw || null, }; }; - const { visibleText, cotText } = splitChainOfThought(textContent); - const imagePaths = extractImagePaths(visibleText); - const displayText = - imagePaths.length > 0 ? removeImagePathsFromText(visibleText, imagePaths) : visibleText; + const { displayText, cotText } = splitChainOfThought(textContent); const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]); const toolRequests = getToolRequests(message); diff --git a/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx b/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx index cb7521d10e0..da3dca5a101 100644 --- a/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx +++ b/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx @@ -271,7 +271,7 @@ const AppSidebar: React.FC = ({ currentPath }) => { let isPolling = false; const handleSessionCreated = (event: Event) => { - const { session } = (event as CustomEvent<{ session?: Session }>).detail; + const { session } = (event as CustomEvent<{ session?: Session }>).detail || {}; // If session data is provided, add it immediately to the sidebar // This is for displaying sessions that won't be returned by the API due to not having messages yet if (session) { diff --git a/ui/desktop/src/components/Hub.tsx b/ui/desktop/src/components/Hub.tsx index 814cdb23d93..583e9908f0f 100644 --- a/ui/desktop/src/components/Hub.tsx +++ b/ui/desktop/src/components/Hub.tsx @@ -40,8 +40,8 @@ export default function Hub({ const [workingDir, setWorkingDir] = useState(getInitialWorkingDir()); const [isCreatingSession, setIsCreatingSession] = useState(false); - const handleSubmit = async (userMessage: string, _images: ImageData[]) => { - if (userMessage.trim() && !isCreatingSession) { + const handleSubmit = async (userMessage: string, images: ImageData[]) => { + if ((images.length > 0 || userMessage.trim()) && !isCreatingSession) { const extensionConfigs = getExtensionConfigsWithOverrides(extensionsList); clearExtensionOverrides(); setIsCreatingSession(true); @@ -55,14 +55,14 @@ export default function Hub({ window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED)); window.dispatchEvent( new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, { - detail: { sessionId: session.id, initialMessage: userMessage }, + detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } }, }) ); setView('pair', { disableAnimation: true, resumeSessionId: session.id, - initialMessage: userMessage, + initialMessage: { msg: userMessage, images }, }); } catch (error) { console.error('Failed to create session:', error); diff --git a/ui/desktop/src/components/ImagePreview.tsx b/ui/desktop/src/components/ImagePreview.tsx index 29e0aff33c9..66d33200896 100644 --- a/ui/desktop/src/components/ImagePreview.tsx +++ b/ui/desktop/src/components/ImagePreview.tsx @@ -1,88 +1,34 @@ -import { useState, useEffect } from 'react'; +import { useState } from 'react'; interface ImagePreviewProps { src: string; - alt?: string; - className?: string; } export default function ImagePreview({ src, - alt = 'Pasted image', - className = '', }: ImagePreviewProps) { const [isExpanded, setIsExpanded] = useState(false); const [error, setError] = useState(false); - const [isLoading, setIsLoading] = useState(true); - const [imageData, setImageData] = useState(null); - - useEffect(() => { - const loadImage = async () => { - try { - // Use the IPC handler to get the image data - const data = await window.electron.getTempImage(src); - if (data) { - setImageData(data); - setIsLoading(false); - } else { - setError(true); - setIsLoading(false); - } - } catch (err) { - console.error('Error loading image:', err); - setError(true); - setIsLoading(false); - } - }; - - loadImage(); - }, [src]); - - const handleError = () => { - setError(true); - setIsLoading(false); - }; - - const toggleExpand = () => { - if (!error) { - setIsExpanded(!isExpanded); - } - }; - - // Validate that this is a safe file path (should contain goose-pasted-images) - if (!src.includes('goose-pasted-images')) { - return
Invalid image path: {src}
; - } if (error) { - return
Unable to load image: {src}
; + return
Unable to load image
; } return ( -
- {isLoading && ( -
- Loading... -
- )} - {imageData && ( - {alt} - )} - {isExpanded && !error && !isLoading && imageData && ( -
Click to collapse
- )} - {!isExpanded && !error && !isLoading && imageData && ( -
Click to expand
- )} +
+ goose image setError(true)} + onClick={() => setIsExpanded(!isExpanded)} + className={`rounded border border-borderSubtle cursor-pointer hover:border-borderStandard transition-all ${ + isExpanded ? 'max-w-full max-h-96' : 'max-h-40 max-w-40' + }`} + style={{ objectFit: 'contain' }} + /> +
+ Click to {isExpanded ? 'collapse' : 'expand'} +
); } diff --git a/ui/desktop/src/components/Layout/AppLayout.tsx b/ui/desktop/src/components/Layout/AppLayout.tsx index 81bae98e795..e2275e0e9ea 100644 --- a/ui/desktop/src/components/Layout/AppLayout.tsx +++ b/ui/desktop/src/components/Layout/AppLayout.tsx @@ -7,9 +7,10 @@ import { Button } from '../ui/button'; import { Sidebar, SidebarInset, SidebarProvider, SidebarTrigger, useSidebar } from '../ui/sidebar'; import ChatSessionsContainer from '../ChatSessionsContainer'; import { useChatContext } from '../../contexts/ChatContext'; +import { ImageData } from '../../types/message'; interface AppLayoutContentProps { - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; + activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; } const AppLayoutContent: React.FC = ({ activeSessions }) => { @@ -126,7 +127,7 @@ const AppLayoutContent: React.FC = ({ activeSessions }) = }; interface AppLayoutProps { - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; + activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; } export const AppLayout: React.FC = ({ activeSessions }) => { diff --git a/ui/desktop/src/components/UserMessage.tsx b/ui/desktop/src/components/UserMessage.tsx index daf71dcf803..389b025d34d 100644 --- a/ui/desktop/src/components/UserMessage.tsx +++ b/ui/desktop/src/components/UserMessage.tsx @@ -1,8 +1,7 @@ -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import ImagePreview from './ImagePreview'; -import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils'; import MarkdownContent from './MarkdownContent'; -import { getTextContent } from '../types/message'; +import { getTextAndImageContent } from '../types/message'; import { Message } from '../api'; import MessageCopyLink from './MessageCopyLink'; import { formatMessageTimestamp } from '../utils/timeUtils'; @@ -21,35 +20,23 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro const [editContent, setEditContent] = useState(''); const [error, setError] = useState(null); - // Extract text content from the message - const textContent = getTextContent(message); - - // Extract image paths from the message - const imagePaths = extractImagePaths(textContent); - - // Remove image paths from text for display - memoized for performance - const displayText = useMemo( - () => removeImagePathsFromText(textContent, imagePaths), - [textContent, imagePaths] - ); - - // Memoize the timestamp - const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]); + const {textContent, imagePaths} = getTextAndImageContent(message); + const timestamp = formatMessageTimestamp(message.created); // Effect to handle message content changes and ensure persistence useEffect(() => { // If we're not editing, update the edit content to match the current message if (!isEditing) { - setEditContent(displayText); + setEditContent(textContent); } - }, [message.content, displayText, message.id, isEditing]); + }, [message.content, textContent, message.id, isEditing]); // Initialize edit mode with current message content const initializeEditMode = useCallback(() => { - setEditContent(displayText); + setEditContent(textContent); setError(null); - window.electron.logInfo(`Entering edit mode with content: ${displayText}`); - }, [displayText]); + window.electron.logInfo(`Entering edit mode with content: ${textContent}`); + }, [textContent]); // Handle edit button click const handleEditClick = useCallback(() => { @@ -93,7 +80,7 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro setIsEditing(false); - if (editContent.trim() === displayText.trim()) { + if (editContent.trim() === textContent.trim()) { return; } @@ -101,16 +88,16 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro onMessageUpdate(message.id, editContent, editType); } }, - [editContent, displayText, onMessageUpdate, message.id] + [editContent, textContent, onMessageUpdate, message.id] ); // Handle cancel action const handleCancel = useCallback(() => { window.electron.logInfo('Cancel clicked - reverting to original content'); setIsEditing(false); - setEditContent(displayText); // Reset to original content + setEditContent(textContent); // Reset to original content setError(null); - }, [displayText]); + }, [textContent]); // Handle keyboard events for accessibility const handleKeyDown = useCallback( @@ -210,7 +197,7 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
@@ -239,14 +226,14 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro } }} className="flex items-center gap-1 text-xs text-text-subtle hover:cursor-pointer hover:text-text-prominent transition-all duration-200 opacity-0 group-hover:opacity-100 -translate-y-4 group-hover:translate-y-0 focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50 rounded" - aria-label={`Edit message: ${displayText.substring(0, 50)}${displayText.length > 50 ? '...' : ''}`} + aria-label={`Edit message: ${textContent.substring(0, 50)}${textContent.length > 50 ? '...' : ''}`} aria-expanded={isEditing} title="Edit message" > Edit - +
diff --git a/ui/desktop/src/components/sessions/SessionViewComponents.tsx b/ui/desktop/src/components/sessions/SessionViewComponents.tsx index b0188ddc9f3..d371365b7e3 100644 --- a/ui/desktop/src/components/sessions/SessionViewComponents.tsx +++ b/ui/desktop/src/components/sessions/SessionViewComponents.tsx @@ -7,12 +7,11 @@ import MarkdownContent from '../MarkdownContent'; import ToolCallWithResponse from '../ToolCallWithResponse'; import ImagePreview from '../ImagePreview'; import { - getTextContent, + getTextAndImageContent, ToolRequestMessageContent, ToolResponseMessageContent, } from '../../types/message'; import { formatMessageTimestamp } from '../../utils/timeUtils'; -import { extractImagePaths, removeImagePathsFromText } from '../../utils/imageUtils'; import { Message } from '../../api'; /** @@ -82,15 +81,7 @@ export const SessionMessages: React.FC = ({ ) : messages?.length > 0 ? ( messages .map((message, index) => { - const textContent = getTextContent(message); - // Extract image paths from the message - const imagePaths = extractImagePaths(textContent); - - // Remove image paths from text for display - const displayText = - imagePaths.length > 0 - ? removeImagePathsFromText(textContent, imagePaths) - : textContent; + const {textContent, imagePaths} = getTextAndImageContent(message); // Get tool requests from the message const toolRequests = message.content @@ -128,12 +119,11 @@ export const SessionMessages: React.FC = ({
- {/* Text content */} - {displayText && ( + {textContent && (
0 || imagePaths.length > 0 ? 'mb-4' : ''}`} > - +
)} diff --git a/ui/desktop/src/hooks/useAutoSubmit.ts b/ui/desktop/src/hooks/useAutoSubmit.ts index b78bb5cc615..837d6e96058 100644 --- a/ui/desktop/src/hooks/useAutoSubmit.ts +++ b/ui/desktop/src/hooks/useAutoSubmit.ts @@ -18,7 +18,7 @@ interface UseAutoSubmitProps { session: Session | undefined; messages: Message[]; chatState: ChatState; - initialMessage: string | undefined; + initialMessage: { msg: string; images: ImageData[] } | undefined; handleSubmit: (message: string, images: ImageData[]) => void; } @@ -69,7 +69,7 @@ export function useAutoSubmit({ // Hub always creates new sessions, so message_count will be 0 if (initialMessage && session.message_count === 0 && messages.length === 0) { hasAutoSubmittedRef.current = true; - handleSubmit(initialMessage, []); + handleSubmit(initialMessage.msg, initialMessage.images); clearInitialMessage(); return; } @@ -77,7 +77,7 @@ export function useAutoSubmit({ // Scenario 2: Forked session with edited message if (shouldStartAgent && initialMessage) { hasAutoSubmittedRef.current = true; - handleSubmit(initialMessage, []); + handleSubmit(initialMessage.msg, initialMessage.images); clearInitialMessage(); return; } diff --git a/ui/desktop/src/main.ts b/ui/desktop/src/main.ts index 835bd595f59..57c87c04cac 100644 --- a/ui/desktop/src/main.ts +++ b/ui/desktop/src/main.ts @@ -1575,79 +1575,6 @@ ipcMain.handle('save-data-url-to-temp', async (_event, dataUrl: string, uniqueId } }); -// IPC handler to serve temporary image files -ipcMain.handle('get-temp-image', async (_event, filePath: string) => { - console.log(`[Main] Received get-temp-image for path: ${filePath}`); - - // Input validation - if (!filePath || typeof filePath !== 'string') { - console.warn('[Main] Invalid file path provided for image serving'); - return null; - } - - // Ensure the path is within the designated temp directory - const resolvedPath = path.resolve(filePath); - const resolvedTempDir = path.resolve(gooseTempDir); - - if (!resolvedPath.startsWith(resolvedTempDir + path.sep)) { - console.warn(`[Main] Attempted to access file outside designated temp directory: ${filePath}`); - return null; - } - - try { - // Check if it's a regular file first, before trying realpath - const stats = await fs.lstat(filePath); - if (!stats.isFile()) { - console.warn(`[Main] Not a regular file, refusing to serve: ${filePath}`); - return null; - } - - // Get the real paths for both the temp directory and the file to handle symlinks properly - let realTempDir: string; - let actualPath = filePath; - - try { - realTempDir = await fs.realpath(gooseTempDir); - const realPath = await fs.realpath(filePath); - - // Double-check that the real path is still within our real temp directory - if (!realPath.startsWith(realTempDir + path.sep)) { - console.warn( - `[Main] Real path is outside designated temp directory: ${realPath} not in ${realTempDir}` - ); - return null; - } - actualPath = realPath; - } catch (realpathError) { - // If realpath fails, use the original path validation - console.log( - `[Main] realpath failed for ${filePath}, using original path validation:`, - realpathError instanceof Error ? realpathError.message : String(realpathError) - ); - } - - // Read the file and return as base64 data URL - const fileBuffer = await fs.readFile(actualPath); - const fileExtension = path.extname(actualPath).toLowerCase().substring(1); - - // Validate file extension - const allowedExtensions = ['png', 'jpg', 'jpeg', 'gif', 'webp']; - if (!allowedExtensions.includes(fileExtension)) { - console.warn(`[Main] Unsupported file extension: ${fileExtension}`); - return null; - } - - const mimeType = fileExtension === 'jpg' ? 'image/jpeg' : `image/${fileExtension}`; - const base64Data = fileBuffer.toString('base64'); - const dataUrl = `data:${mimeType};base64,${base64Data}`; - - console.log(`[Main] Served temp image: ${filePath}`); - return dataUrl; - } catch (error) { - console.error(`[Main] Failed to serve temp image: ${filePath}`, error); - return null; - } -}); ipcMain.on('delete-temp-file', async (_event, filePath: string) => { console.log(`[Main] Received delete-temp-file for path: ${filePath}`); diff --git a/ui/desktop/src/preload.ts b/ui/desktop/src/preload.ts index 3ac6ce0b843..e8fb4a025fc 100644 --- a/ui/desktop/src/preload.ts +++ b/ui/desktop/src/preload.ts @@ -121,8 +121,6 @@ type ElectronAPI = { deleteTempFile: (filePath: string) => void; // Function for opening external URLs securely openExternal: (url: string) => Promise; - // Function to serve temp images - getTempImage: (filePath: string) => Promise; // Update-related functions getVersion: () => string; checkForUpdates: () => Promise<{ updateInfo: unknown; error: string | null }>; @@ -244,9 +242,6 @@ const electronAPI: ElectronAPI = { openExternal: (url: string): Promise => { return ipcRenderer.invoke('open-external', url); }, - getTempImage: (filePath: string): Promise => { - return ipcRenderer.invoke('get-temp-image', filePath); - }, getVersion: (): string => { return config.GOOSE_VERSION || ipcRenderer.sendSync('get-app-version') || ''; }, diff --git a/ui/desktop/src/sessions.ts b/ui/desktop/src/sessions.ts index 1ef635ed712..d3e87967ba6 100644 --- a/ui/desktop/src/sessions.ts +++ b/ui/desktop/src/sessions.ts @@ -88,9 +88,11 @@ export async function startNewSession( // Include session data so sidebar can add it immediately (before it has messages) window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED, { detail: { session } })); + const initialMessage = initialText ? { msg: initialText, images: [] } : undefined; + const eventDetail = { sessionId: session.id, - initialMessage: initialText, + initialMessage, }; window.dispatchEvent( @@ -101,7 +103,7 @@ export async function startNewSession( setView('pair', { disableAnimation: true, - initialMessage: initialText, + initialMessage, resumeSessionId: session.id, }); return session; diff --git a/ui/desktop/src/types/message.ts b/ui/desktop/src/types/message.ts index 7e54514c3f2..2384e079dcc 100644 --- a/ui/desktop/src/types/message.ts +++ b/ui/desktop/src/types/message.ts @@ -66,13 +66,22 @@ export function generateMessageId(): string { return Math.random().toString(36).substring(2, 10); } -export function getTextContent(message: Message): string { - return message.content - .map((content) => { - if (content.type === 'text') return content.text; - return ''; - }) - .join(''); +export function getTextAndImageContent(message: Message): { + textContent: string; + imagePaths: string[]; +} { + let textContent = ''; + const imagePaths: string[] = []; + + for (const content of message.content) { + if (content.type === 'text') { + textContent += content.text; + } else if (content.type === 'image') { + imagePaths.push(`data:${content.mimeType};base64,${content.data}`); + } + } + + return { textContent, imagePaths }; } export function getToolRequests(message: Message): (ToolRequest & { type: 'toolRequest' })[] { diff --git a/ui/desktop/src/utils/imageUtils.ts b/ui/desktop/src/utils/imageUtils.ts deleted file mode 100644 index 83bb7338c1b..00000000000 --- a/ui/desktop/src/utils/imageUtils.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Utility functions for detecting and handling image paths in messages - */ - -/** - * Extracts image file paths from a message text - * Looks for paths that match the pattern of pasted images from the temp directory - * - * @param text The message text to extract image paths from - * @returns An array of image file paths found in the message - */ -export function extractImagePaths(text: string): string[] { - if (!text) return []; - - // Match paths that look like pasted image paths from the temp directory - // Pattern: /path/to/goose-pasted-images/pasted-img-TIMESTAMP-RANDOM.ext - // This regex looks for: - // - Word boundary or start of string - // - A path containing "goose-pasted-images" - // - Followed by a filename starting with "pasted-" - // - Ending with common image extensions - // - Word boundary or end of string - const regex = - /(?:^|\s)((?:[^\s]*\/)?goose-pasted-images\/pasted-[^\s]+\.(png|jpg|jpeg|gif|webp))(?=\s|$)/gi; - - const matches = []; - let match; - - while ((match = regex.exec(text)) !== null) { - matches.push(match[1]); - } - - return matches; -} - -/** - * Removes image paths from the text - * - * @param text The original text - * @param imagePaths Array of image paths to remove - * @returns Text with image paths removed - */ -export function removeImagePathsFromText(text: string, imagePaths: string[]): string { - if (!text || imagePaths.length === 0) return text; - - let result = text; - - // Remove each image path from the text - imagePaths.forEach((path) => { - // Escape special regex characters in the path - const escapedPath = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - // Create a regex that matches the path with optional surrounding whitespace - const pathRegex = new RegExp(`(^|\\s)${escapedPath}(?=\\s|$)`, 'g'); - result = result.replace(pathRegex, '$1'); - }); - - // Clean up any extra whitespace - return result.replace(/\s+/g, ' ').trim(); -} diff --git a/ui/desktop/src/utils/navigationUtils.ts b/ui/desktop/src/utils/navigationUtils.ts index bb846291bf4..d8a769f985c 100644 --- a/ui/desktop/src/utils/navigationUtils.ts +++ b/ui/desktop/src/utils/navigationUtils.ts @@ -1,5 +1,6 @@ import { NavigateFunction } from 'react-router-dom'; -import { Recipe } from '../api/types.gen'; +import { Recipe } from '../api'; +import { ImageData } from '../types/message'; export type View = | 'welcome' @@ -29,7 +30,7 @@ export type ViewOptions = { parentView?: View; parentViewOptions?: ViewOptions; disableAnimation?: boolean; - initialMessage?: string; + initialMessage?: {msg: string, images: ImageData[]}; shareToken?: string; resumeSessionId?: string; pendingScheduleDeepLink?: string; diff --git a/ui/desktop/src/utils/toolCallChaining.ts b/ui/desktop/src/utils/toolCallChaining.ts index d676da87443..854557f141c 100644 --- a/ui/desktop/src/utils/toolCallChaining.ts +++ b/ui/desktop/src/utils/toolCallChaining.ts @@ -1,4 +1,4 @@ -import { getToolRequests, getTextContent, getToolResponses } from '../types/message'; +import { getToolRequests, getTextAndImageContent, getToolResponses } from '../types/message'; import { Message } from '../api'; export function identifyConsecutiveToolCalls(messages: Message[]): number[][] { @@ -9,7 +9,7 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] { const message = messages[i]; const toolRequests = getToolRequests(message); const toolResponses = getToolResponses(message); - const textContent = getTextContent(message); + const {textContent} = getTextAndImageContent(message); const hasText = textContent.trim().length > 0; if (toolResponses.length > 0 && toolRequests.length === 0) { @@ -47,15 +47,6 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] { return chains; } -export function shouldHideMessage(messageIndex: number, chains: number[][]): boolean { - for (const chain of chains) { - if (chain.includes(messageIndex)) { - return chain[0] !== messageIndex; - } - } - return false; -} - export function shouldHideTimestamp(messageIndex: number, chains: number[][]): boolean { for (const chain of chains) { if (chain.includes(messageIndex)) { @@ -69,7 +60,3 @@ export function shouldHideTimestamp(messageIndex: number, chains: number[][]): b export function isInChain(messageIndex: number, chains: number[][]): boolean { return chains.some((chain) => chain.includes(messageIndex)); } - -export function getChainForMessage(messageIndex: number, chains: number[][]): number[] | null { - return chains.find((chain) => chain.includes(messageIndex)) || null; -} From 0b16387a5ad65a9952245468d9394e75a6c61800 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Wed, 21 Jan 2026 20:31:03 -0500 Subject: [PATCH 05/13] Update ui/desktop/src/components/ChatInput.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ui/desktop/src/components/ChatInput.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index bb67de5419a..cb0ec31a2f9 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -1141,6 +1141,7 @@ export default function ChatInput({ // Check if we're at the image limit if (pastedImages.length >= MAX_IMAGES_PER_MESSAGE) { console.warn(`Maximum ${MAX_IMAGES_PER_MESSAGE} images per message`); + toastError(`You can only attach up to ${MAX_IMAGES_PER_MESSAGE} images per message.`); return; } From 6a42db9d09c572f6903452985bbcaab9a39f4fb4 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 15:47:37 -0500 Subject: [PATCH 06/13] Merge? --- ui/desktop/src/App.tsx | 14 +- ui/desktop/src/components/ChatInput.tsx | 331 +++++++----------- .../src/components/ChatSessionsContainer.tsx | 5 +- ui/desktop/src/components/GooseMessage.tsx | 6 +- ui/desktop/src/components/ImagePreview.tsx | 4 +- .../src/components/Layout/AppLayout.tsx | 10 +- ui/desktop/src/components/UserMessage.tsx | 5 +- .../sessions/SessionViewComponents.tsx | 9 +- ui/desktop/src/hooks/useFileDrop.ts | 71 +++- ui/desktop/src/types/message.ts | 8 +- ui/desktop/src/utils/navigationUtils.ts | 2 +- ui/desktop/src/utils/toolCallChaining.ts | 2 +- 12 files changed, 232 insertions(+), 235 deletions(-) diff --git a/ui/desktop/src/App.tsx b/ui/desktop/src/App.tsx index 2031ee7ccd5..5f0ef0d591a 100644 --- a/ui/desktop/src/App.tsx +++ b/ui/desktop/src/App.tsx @@ -68,8 +68,13 @@ const HubRouteWrapper = () => { const PairRouteWrapper = ({ activeSessions, }: { - activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; - setActiveSessions: (sessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>) => void; + activeSessions: Array<{ + sessionId: string; + initialMessage?: { msg: string; images: ImageData[] }; + }>; + setActiveSessions: ( + sessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }> + ) => void; }) => { const { extensionsList } = useConfig(); const location = useLocation(); @@ -360,7 +365,10 @@ export function AppInner() { useEffect(() => { const handleAddActiveSession = (event: Event) => { const { sessionId, initialMessage } = ( - event as CustomEvent<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }> + event as CustomEvent<{ + sessionId: string; + initialMessage?: { msg: string; images: ImageData[] }; + }> ).detail; setActiveSessions((prev) => { diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index cb0ec31a2f9..6041473243a 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -25,7 +25,7 @@ import { COST_TRACKING_ENABLED, VOICE_DICTATION_ELEVENLABS_ENABLED } from '../up import { CostTracker } from './bottom_menu/CostTracker'; import { DroppedFile, useFileDrop } from '../hooks/useFileDrop'; import { Recipe } from '../recipe'; -import {MessageQueue, QueuedMessage} from './MessageQueue'; +import { MessageQueue, QueuedMessage } from './MessageQueue'; import { detectInterruption } from '../utils/interruptionDetector'; import { DiagnosticsModal } from './ui/Diagnostics'; import { getSession, Message } from '../api'; @@ -45,14 +45,12 @@ import { ImageData } from '../types/message'; interface PastedImage { id: string; - dataUrl: string; // For immediate preview - filePath?: string; // Path on filesystem after saving + dataUrl: string; isLoading: boolean; error?: string; } -// Constants for image handling -const MAX_IMAGES_PER_MESSAGE = 5; +const MAX_IMAGES_PER_MESSAGE = 10; // Constants for token and tool alerts const TOKEN_LIMIT_DEFAULT = 128000; // fallback for custom models that the backend doesn't know about @@ -213,9 +211,7 @@ export default function ChatInput({ if (shouldProcessQueue) { const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); - handleSubmit( - nextMessage.content, nextMessage.images - ); + handleSubmit(nextMessage.content, nextMessage.images); setQueuedMessages((prev) => { const newQueue = prev.slice(1); // If queue becomes empty after processing, clear the paused state @@ -301,15 +297,7 @@ export default function ChatInput({ useEffect(() => { setValue(initialValue); setDisplayValue(initialValue); - setPastedImages((currentPastedImages) => { - currentPastedImages.forEach((img) => { - if (img.filePath) { - window.electron.deleteTempFile(img.filePath); - } - }); - return []; - }); - + setPastedImages([]); setHistoryIndex(-1); setIsInGlobalHistory(false); setHasUserTyped(false); @@ -358,43 +346,9 @@ export default function ChatInput({ }; const handleRemovePastedImage = (idToRemove: string) => { - const imageToRemove = pastedImages.find((img) => img.id === idToRemove); - if (imageToRemove?.filePath) { - window.electron.deleteTempFile(imageToRemove.filePath); - } setPastedImages((currentImages) => currentImages.filter((img) => img.id !== idToRemove)); }; - const handleRetryImageSave = async (imageId: string) => { - const imageToRetry = pastedImages.find((img) => img.id === imageId); - if (!imageToRetry || !imageToRetry.dataUrl) return; - - // Set the image to loading state - setPastedImages((prev) => - prev.map((img) => (img.id === imageId ? { ...img, isLoading: true, error: undefined } : img)) - ); - - try { - const result = await window.electron.saveDataUrlToTemp(imageToRetry.dataUrl, imageId); - setPastedImages((prev) => - prev.map((img) => - img.id === result.id - ? { ...img, filePath: result.filePath, error: result.error, isLoading: false } - : img - ) - ); - } catch (err) { - console.error('Error retrying image save:', err); - setPastedImages((prev) => - prev.map((img) => - img.id === imageId - ? { ...img, error: 'Failed to save image via Electron.', isLoading: false } - : img - ) - ); - } - }; - useEffect(() => { if (textAreaRef.current) { textAreaRef.current.focus(); @@ -529,20 +483,6 @@ export default function ChatInput({ // Cleanup effect for component unmount - prevent memory leaks useEffect(() => { return () => { - // Clear any pending timeouts from image processing - setPastedImages((currentImages) => { - currentImages.forEach((img) => { - if (img.filePath) { - try { - window.electron.deleteTempFile(img.filePath); - } catch (error) { - console.error('Error deleting temp file:', error); - } - } - }); - return []; - }); - // Clear all tracked timeouts // eslint-disable-next-line react-hooks/exhaustive-deps const timeouts = timeoutRefsRef.current; @@ -653,7 +593,7 @@ export default function ChatInput({ const maxDimension = 1024; let width = img.width; let height = img.height; - + if (width > maxDimension || height > maxDimension) { if (width > height) { height = Math.floor((height * maxDimension) / width); @@ -663,20 +603,20 @@ export default function ChatInput({ height = maxDimension; } } - + // Create canvas and resize const canvas = document.createElement('canvas'); canvas.width = width; canvas.height = height; const ctx = canvas.getContext('2d'); - + if (!ctx) { reject(new Error('Failed to get canvas context')); return; } - + ctx.drawImage(img, 0, 0, width, height); - + // Convert to JPEG with 0.85 quality const compressedDataUrl = canvas.toDataURL('image/jpeg', 0.85); resolve(compressedDataUrl); @@ -735,36 +675,12 @@ export default function ChatInput({ reader.onload = async (e) => { const dataUrl = e.target?.result as string; if (dataUrl) { - try { - // Compress the image - const compressedDataUrl = await compressImageDataUrl(dataUrl); - - // Update the image with the compressed data URL - setPastedImages((prev) => - prev.map((img) => (img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: false } : img)) - ); - } catch (compressionError) { - console.error('Error compressing image:', compressionError); - setPastedImages((prev) => - prev.map((img) => - img.id === imageId - ? { - ...img, - dataUrl, - isLoading: false, - error: `Image too large and could not be compressed. Try a smaller image.`, - } - : img - ) - ); - - // Remove the error message after 5 seconds - const timeoutId = setTimeout(() => { - setPastedImages((prev) => prev.filter((img) => img.id !== imageId)); - timeoutRefsRef.current.delete(timeoutId); - }, 5000); - timeoutRefsRef.current.add(timeoutId); - } + const compressedDataUrl = await compressImageDataUrl(dataUrl); + setPastedImages((prev) => + prev.map((img) => + img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: false } : img + ) + ); } }; reader.onerror = () => { @@ -884,8 +800,42 @@ export default function ChatInput({ return false; } + // Extract base64 image data from pasted images + const pastedImageData: ImageData[] = pastedImages + .filter((img) => img.dataUrl && !img.error && !img.isLoading) + .map((img) => { + const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], + mimeType: matches[1], + }; + } + return null; + }) + .filter((img): img is ImageData => img !== null); + + // Extract base64 image data from dropped images + const droppedImageData: ImageData[] = allDroppedFiles + .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) + .map((file) => { + const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], + mimeType: matches[1], + }; + } + return null; + }) + .filter((img): img is ImageData => img !== null); + + // Combine all image data + const imageData = [...pastedImageData, ...droppedImageData]; + + // Get paths from non-image dropped files only const droppedFilePaths = allDroppedFiles - .filter((file) => !file.error && !file.isLoading) + .filter((file) => !file.isImage && !file.error && !file.isLoading) .map((file) => file.path); let contentToQueue = displayValue.trim(); @@ -907,7 +857,7 @@ export default function ChatInput({ id: Date.now().toString() + Math.random().toString(36).substr(2, 9), content: contentToQueue, timestamp: Date.now(), - images: [], + images: imageData, }; // Add the interruption message to the front of the queue so it gets sent first @@ -929,7 +879,7 @@ export default function ChatInput({ id: Date.now().toString() + Math.random().toString(36).substr(2, 9), content: contentToQueue, timestamp: Date.now(), - images: [], + images: imageData, }; setQueuedMessages((prev) => { const newQueue = [...prev, newMessage]; @@ -961,7 +911,7 @@ export default function ChatInput({ const performSubmit = useCallback( (text?: string) => { // Extract base64 image data from pasted images (for direct image content) - const imageData: import('../types/message').ImageData[] = pastedImages + const pastedImageData: ImageData[] = pastedImages .filter((img) => img.dataUrl && !img.error && !img.isLoading) .map((img) => { // Extract base64 data and mime type from data URL @@ -975,16 +925,35 @@ export default function ChatInput({ } return null; }) - .filter((img): img is import('../types/message').ImageData => img !== null); + .filter((img): img is ImageData => img !== null); + + // Extract base64 image data from dropped images (for direct image content) + const droppedImageData: ImageData[] = allDroppedFiles + .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) + .map((file) => { + // Extract base64 data and mime type from data URL + const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], // base64 data + mimeType: matches[1], // mime type + }; + } + return null; + }) + .filter((img): img is ImageData => img !== null); + + // Combine all image data + const imageData = [...pastedImageData, ...droppedImageData]; - // Get paths from all dropped files + // Get paths from non-image dropped files only const droppedFilePaths = allDroppedFiles - .filter((file) => !file.error && !file.isLoading) + .filter((file) => !file.isImage && !file.error && !file.isLoading) .map((file) => file.path); let textToSend = text ?? displayValue.trim(); - // Add dropped file paths to text + // Add non-image dropped file paths to text if (droppedFilePaths.length > 0) { const pathsString = droppedFilePaths.join(' '); textToSend = textToSend ? `${textToSend} ${pathsString}` : pathsString; @@ -1131,91 +1100,66 @@ export default function ChatInput({ if (!files || files.length === 0) return; setIsFilePickerOpen(true); - try { - const file = files[0]; - const isImage = file.type.startsWith('image/'); - - if (isImage) { - trackFileAttached('file'); - - // Check if we're at the image limit - if (pastedImages.length >= MAX_IMAGES_PER_MESSAGE) { - console.warn(`Maximum ${MAX_IMAGES_PER_MESSAGE} images per message`); - toastError(`You can only attach up to ${MAX_IMAGES_PER_MESSAGE} images per message.`); - return; - } + const file = files[0]; + const isImage = file.type.startsWith('image/'); + + if (isImage) { + trackFileAttached('file'); + + if (pastedImages.length >= MAX_IMAGES_PER_MESSAGE) { + console.warn(`Maximum ${MAX_IMAGES_PER_MESSAGE} images per message`); + setIsFilePickerOpen(false); + return; + } + + const uniqueId = `upload-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; - // Create a unique ID for this image - const uniqueId = `upload-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; - - // Add a loading placeholder - setPastedImages(prev => [...prev, { + setPastedImages((prev) => [ + ...prev, + { id: uniqueId, dataUrl: '', isLoading: true, - error: undefined - }]); - - try { - // Read the file using FileReader - const reader = new FileReader(); - reader.onload = async (evt) => { - const dataUrl = evt.target?.result as string; - if (dataUrl) { - try { - // Compress the image - const compressedDataUrl = await compressImageDataUrl(dataUrl); - - // Update the image with the compressed data URL - setPastedImages(prev => prev.map(img => - img.id === uniqueId - ? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined } - : img - )); - } catch (compressionError) { - const errorMessage = compressionError instanceof Error - ? compressionError.message - : 'Failed to compress image'; - setPastedImages(prev => prev.map(img => - img.id === uniqueId - ? { ...img, isLoading: false, error: errorMessage } - : img - )); - } - } - }; - reader.onerror = () => { - setPastedImages(prev => prev.map(img => - img.id === uniqueId - ? { ...img, isLoading: false, error: 'Failed to read image file' } + error: undefined, + }, + ]); + + const reader = new FileReader(); + reader.onload = async (evt) => { + const dataUrl = evt.target?.result as string; + if (dataUrl) { + const compressedDataUrl = await compressImageDataUrl(dataUrl); + setPastedImages((prev) => + prev.map((img) => + img.id === uniqueId + ? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined } : img - )); - }; - reader.readAsDataURL(file); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : 'Failed to load image'; - setPastedImages(prev => prev.map(img => - img.id === uniqueId - ? { ...img, isLoading: false, error: errorMessage } - : img - )); + ) + ); } - } else { - // For non-image files, get the path and add to text - trackFileAttached('file'); - const path = window.electron.getPathForFile(file); - const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path; - setDisplayValue(newValue); - setValue(newValue); - } - - textAreaRef.current?.focus(); - } finally { - setIsFilePickerOpen(false); - // Reset the input so the same file can be selected again - if (e.target) { - e.target.value = ''; - } + }; + reader.onerror = () => { + setPastedImages((prev) => + prev.map((img) => + img.id === uniqueId + ? { ...img, isLoading: false, error: 'Failed to read image file' } + : img + ) + ); + }; + reader.readAsDataURL(file); + } else { + trackFileAttached('file'); + const path = window.electron.getPathForFile(file); + const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path; + setDisplayValue(newValue); + setValue(newValue); + } + + textAreaRef.current?.focus(); + setIsFilePickerOpen(false); + if (e.target) { + e.target.value = ''; } }; @@ -1554,20 +1498,9 @@ export default function ChatInput({ )} {img.error && !img.isLoading && (
-

+

{img.error.substring(0, 50)}

- {img.dataUrl && ( - - )}
)} {!img.isLoading && ( diff --git a/ui/desktop/src/components/ChatSessionsContainer.tsx b/ui/desktop/src/components/ChatSessionsContainer.tsx index 0047bd7f06a..92aa571828b 100644 --- a/ui/desktop/src/components/ChatSessionsContainer.tsx +++ b/ui/desktop/src/components/ChatSessionsContainer.tsx @@ -5,7 +5,10 @@ import { ImageData } from '../types/message'; interface ChatSessionsContainerProps { setChat: (chat: ChatType) => void; - activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; + activeSessions: Array<{ + sessionId: string; + initialMessage?: { msg: string; images: ImageData[] }; + }>; } /** diff --git a/ui/desktop/src/components/GooseMessage.tsx b/ui/desktop/src/components/GooseMessage.tsx index f916030e8a2..c6b593f50b0 100644 --- a/ui/desktop/src/components/GooseMessage.tsx +++ b/ui/desktop/src/components/GooseMessage.tsx @@ -44,7 +44,7 @@ export default function GooseMessage({ }: GooseMessageProps) { const contentRef = useRef(null); - let {textContent, imagePaths} = getTextAndImageContent(message); + let { textContent, imagePaths } = getTextAndImageContent(message); const splitChainOfThought = (text: string): { displayText: string; cotText: string | null } => { const regex = /([\s\S]*?)<\/think>/i; @@ -54,10 +54,10 @@ export default function GooseMessage({ } const cotRaw = match[1].trim(); - const visibleText = text.replace(regex, '').trim(); + const displayText = text.replace(regex, '').trim(); return { - displayText: visibleText, + displayText, cotText: cotRaw || null, }; }; diff --git a/ui/desktop/src/components/ImagePreview.tsx b/ui/desktop/src/components/ImagePreview.tsx index 66d33200896..c361ae62ef9 100644 --- a/ui/desktop/src/components/ImagePreview.tsx +++ b/ui/desktop/src/components/ImagePreview.tsx @@ -4,9 +4,7 @@ interface ImagePreviewProps { src: string; } -export default function ImagePreview({ - src, -}: ImagePreviewProps) { +export default function ImagePreview({ src }: ImagePreviewProps) { const [isExpanded, setIsExpanded] = useState(false); const [error, setError] = useState(false); diff --git a/ui/desktop/src/components/Layout/AppLayout.tsx b/ui/desktop/src/components/Layout/AppLayout.tsx index e2275e0e9ea..c895d6c0df5 100644 --- a/ui/desktop/src/components/Layout/AppLayout.tsx +++ b/ui/desktop/src/components/Layout/AppLayout.tsx @@ -10,7 +10,10 @@ import { useChatContext } from '../../contexts/ChatContext'; import { ImageData } from '../../types/message'; interface AppLayoutContentProps { - activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; + activeSessions: Array<{ + sessionId: string; + initialMessage?: { msg: string; images: ImageData[] }; + }>; } const AppLayoutContent: React.FC = ({ activeSessions }) => { @@ -127,7 +130,10 @@ const AppLayoutContent: React.FC = ({ activeSessions }) = }; interface AppLayoutProps { - activeSessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }>; + activeSessions: Array<{ + sessionId: string; + initialMessage?: { msg: string; images: ImageData[] }; + }>; } export const AppLayout: React.FC = ({ activeSessions }) => { diff --git a/ui/desktop/src/components/UserMessage.tsx b/ui/desktop/src/components/UserMessage.tsx index 389b025d34d..b691a6ed0a3 100644 --- a/ui/desktop/src/components/UserMessage.tsx +++ b/ui/desktop/src/components/UserMessage.tsx @@ -20,7 +20,7 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro const [editContent, setEditContent] = useState(''); const [error, setError] = useState(null); - const {textContent, imagePaths} = getTextAndImageContent(message); + const { textContent, imagePaths } = getTextAndImageContent(message); const timestamp = formatMessageTimestamp(message.created); // Effect to handle message content changes and ensure persistence @@ -203,11 +203,10 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
- {/* Render images if any */} {imagePaths.length > 0 && (
{imagePaths.map((imagePath, index) => ( - + ))}
)} diff --git a/ui/desktop/src/components/sessions/SessionViewComponents.tsx b/ui/desktop/src/components/sessions/SessionViewComponents.tsx index d371365b7e3..fb8839c118c 100644 --- a/ui/desktop/src/components/sessions/SessionViewComponents.tsx +++ b/ui/desktop/src/components/sessions/SessionViewComponents.tsx @@ -81,7 +81,7 @@ export const SessionMessages: React.FC = ({ ) : messages?.length > 0 ? ( messages .map((message, index) => { - const {textContent, imagePaths} = getTextAndImageContent(message); + const { textContent, imagePaths } = getTextAndImageContent(message); // Get tool requests from the message const toolRequests = message.content @@ -127,15 +127,10 @@ export const SessionMessages: React.FC = ({ )} - {/* Render images if any */} {imagePaths.length > 0 && (
{imagePaths.map((imagePath, imageIndex) => ( - + ))}
)} diff --git a/ui/desktop/src/hooks/useFileDrop.ts b/ui/desktop/src/hooks/useFileDrop.ts index d88e2e77381..bd4c5d19218 100644 --- a/ui/desktop/src/hooks/useFileDrop.ts +++ b/ui/desktop/src/hooks/useFileDrop.ts @@ -6,11 +6,49 @@ export interface DroppedFile { name: string; type: string; isImage: boolean; - dataUrl?: string; // For image previews + dataUrl?: string; // For images: compressed base64 data URL for direct inclusion isLoading?: boolean; error?: string; } +// Helper function to compress image data URLs +const compressImageDataUrl = async (dataUrl: string): Promise => { + return new Promise((resolve, reject) => { + const img = new globalThis.Image(); + img.onload = () => { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + if (!ctx) { + reject(new Error('Failed to get canvas context')); + return; + } + + // Resize to max 1024px on longest side + const maxSize = 1024; + let width = img.width; + let height = img.height; + + if (width > height && width > maxSize) { + height = (height * maxSize) / width; + width = maxSize; + } else if (height > maxSize) { + width = (width * maxSize) / height; + height = maxSize; + } + + canvas.width = width; + canvas.height = height; + ctx.drawImage(img, 0, 0, width, height); + + // Convert to JPEG with 0.85 quality + const compressedDataUrl = canvas.toDataURL('image/jpeg', 0.85); + resolve(compressedDataUrl); + }; + img.onerror = () => reject(new Error('Failed to load image')); + img.src = dataUrl; + }); +}; + export const useFileDrop = () => { const [droppedFiles, setDroppedFiles] = useState([]); const activeReadersRef = useRef>(new Set()); @@ -71,25 +109,42 @@ export const useFileDrop = () => { droppedFileObjects.push(droppedFile); - // For images, generate a preview (only if successfully processed) + // For images, read and compress for direct inclusion if (droppedFile.isImage && !droppedFile.error) { const reader = new FileReader(); activeReadersRef.current.add(reader); - reader.onload = (event) => { + reader.onload = async (event) => { const dataUrl = event.target?.result as string; - setDroppedFiles((prev) => - prev.map((f) => (f.id === droppedFile.id ? { ...f, dataUrl, isLoading: false } : f)) - ); + try { + // Compress the image + const compressedDataUrl = await compressImageDataUrl(dataUrl); + setDroppedFiles((prev) => + prev.map((f) => + f.id === droppedFile.id + ? { ...f, dataUrl: compressedDataUrl, isLoading: false } + : f + ) + ); + } catch (compressionError) { + console.error('Failed to compress image:', file.name, compressionError); + setDroppedFiles((prev) => + prev.map((f) => + f.id === droppedFile.id + ? { ...f, error: 'Failed to compress image', isLoading: false } + : f + ) + ); + } activeReadersRef.current.delete(reader); }; reader.onerror = () => { - console.error('Failed to generate preview for:', file.name); + console.error('Failed to read image:', file.name); setDroppedFiles((prev) => prev.map((f) => f.id === droppedFile.id - ? { ...f, error: 'Failed to load image preview', isLoading: false } + ? { ...f, error: 'Failed to load image', isLoading: false } : f ) ); diff --git a/ui/desktop/src/types/message.ts b/ui/desktop/src/types/message.ts index 2384e079dcc..9539ca7fe55 100644 --- a/ui/desktop/src/types/message.ts +++ b/ui/desktop/src/types/message.ts @@ -14,15 +14,15 @@ export interface ImageData { export function createUserMessage(text: string, images?: ImageData[]): Message { const content: Message['content'] = []; - + // Add text content if present if (text.trim()) { content.push({ type: 'text', text }); } - + // Add image content if present if (images && images.length > 0) { - images.forEach(img => { + images.forEach((img) => { content.push({ type: 'image', data: img.data, @@ -30,7 +30,7 @@ export function createUserMessage(text: string, images?: ImageData[]): Message { }); }); } - + return { id: generateMessageId(), role: 'user', diff --git a/ui/desktop/src/utils/navigationUtils.ts b/ui/desktop/src/utils/navigationUtils.ts index d8a769f985c..e2289e536e8 100644 --- a/ui/desktop/src/utils/navigationUtils.ts +++ b/ui/desktop/src/utils/navigationUtils.ts @@ -30,7 +30,7 @@ export type ViewOptions = { parentView?: View; parentViewOptions?: ViewOptions; disableAnimation?: boolean; - initialMessage?: {msg: string, images: ImageData[]}; + initialMessage?: { msg: string; images: ImageData[] }; shareToken?: string; resumeSessionId?: string; pendingScheduleDeepLink?: string; diff --git a/ui/desktop/src/utils/toolCallChaining.ts b/ui/desktop/src/utils/toolCallChaining.ts index 854557f141c..3b6a5a78539 100644 --- a/ui/desktop/src/utils/toolCallChaining.ts +++ b/ui/desktop/src/utils/toolCallChaining.ts @@ -9,7 +9,7 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] { const message = messages[i]; const toolRequests = getToolRequests(message); const toolResponses = getToolResponses(message); - const {textContent} = getTextAndImageContent(message); + const { textContent } = getTextAndImageContent(message); const hasText = textContent.trim().length > 0; if (toolResponses.length > 0 && toolRequests.length === 0) { From 28ddc98b06df67df18a32ff111225c29ee10b956 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 15:58:41 -0500 Subject: [PATCH 07/13] Simplify --- ui/desktop/src/components/ChatInput.tsx | 57 ++++++------------------- 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index 6041473243a..b7e0ce837e7 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -587,43 +587,21 @@ export default function ChatInput({ }; const compressImageDataUrl = async (dataUrl: string): Promise => { - return new Promise((resolve, reject) => { - const img = new globalThis.Image(); - img.onload = () => { - const maxDimension = 1024; - let width = img.width; - let height = img.height; - - if (width > maxDimension || height > maxDimension) { - if (width > height) { - height = Math.floor((height * maxDimension) / width); - width = maxDimension; - } else { - width = Math.floor((width * maxDimension) / height); - height = maxDimension; - } - } + const res = await fetch(dataUrl); + const blob = await res.blob(); + const bitmap = await globalThis.createImageBitmap(blob); - // Create canvas and resize - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - const ctx = canvas.getContext('2d'); + const maxDim = 1024; + const scale = Math.min(1, maxDim / Math.max(bitmap.width, bitmap.height)); + const width = Math.floor(bitmap.width * scale); + const height = Math.floor(bitmap.height * scale); - if (!ctx) { - reject(new Error('Failed to get canvas context')); - return; - } + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + canvas.getContext('2d')!.drawImage(bitmap, 0, 0, width, height); - ctx.drawImage(img, 0, 0, width, height); - - // Convert to JPEG with 0.85 quality - const compressedDataUrl = canvas.toDataURL('image/jpeg', 0.85); - resolve(compressedDataUrl); - }; - img.onerror = () => reject(new Error('Failed to load image for compression')); - img.src = dataUrl; - }); + return canvas.toDataURL('image/jpeg', 0.85); }; const handlePaste = async (evt: React.ClipboardEvent) => { @@ -800,7 +778,6 @@ export default function ChatInput({ return false; } - // Extract base64 image data from pasted images const pastedImageData: ImageData[] = pastedImages .filter((img) => img.dataUrl && !img.error && !img.isLoading) .map((img) => { @@ -815,7 +792,6 @@ export default function ChatInput({ }) .filter((img): img is ImageData => img !== null); - // Extract base64 image data from dropped images const droppedImageData: ImageData[] = allDroppedFiles .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) .map((file) => { @@ -830,10 +806,8 @@ export default function ChatInput({ }) .filter((img): img is ImageData => img !== null); - // Combine all image data const imageData = [...pastedImageData, ...droppedImageData]; - // Get paths from non-image dropped files only const droppedFilePaths = allDroppedFiles .filter((file) => !file.isImage && !file.error && !file.isLoading) .map((file) => file.path); @@ -910,12 +884,9 @@ export default function ChatInput({ const performSubmit = useCallback( (text?: string) => { - // Extract base64 image data from pasted images (for direct image content) const pastedImageData: ImageData[] = pastedImages .filter((img) => img.dataUrl && !img.error && !img.isLoading) .map((img) => { - // Extract base64 data and mime type from data URL - // Data URL format: data:image/png;base64,iVBORw0KG... const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/); if (matches) { return { @@ -927,11 +898,9 @@ export default function ChatInput({ }) .filter((img): img is ImageData => img !== null); - // Extract base64 image data from dropped images (for direct image content) const droppedImageData: ImageData[] = allDroppedFiles .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) .map((file) => { - // Extract base64 data and mime type from data URL const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/); if (matches) { return { @@ -943,10 +912,8 @@ export default function ChatInput({ }) .filter((img): img is ImageData => img !== null); - // Combine all image data const imageData = [...pastedImageData, ...droppedImageData]; - // Get paths from non-image dropped files only const droppedFilePaths = allDroppedFiles .filter((file) => !file.isImage && !file.error && !file.isLoading) .map((file) => file.path); From 425328142ecbdf5187dab5e34f632919d4cf18ff Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 16:08:46 -0500 Subject: [PATCH 08/13] Have a UserInput type --- ui/desktop/src/App.tsx | 12 +- ui/desktop/src/components/BaseChat.tsx | 4 +- ui/desktop/src/components/ChatInput.tsx | 204 +++++++----------- .../src/components/ChatSessionsContainer.tsx | 4 +- .../src/components/Layout/AppLayout.tsx | 6 +- ui/desktop/src/hooks/useAutoSubmit.ts | 4 +- ui/desktop/src/hooks/useFileDrop.ts | 3 +- ui/desktop/src/types/message.ts | 5 + ui/desktop/src/utils/navigationUtils.ts | 4 +- 9 files changed, 102 insertions(+), 144 deletions(-) diff --git a/ui/desktop/src/App.tsx b/ui/desktop/src/App.tsx index 5f0ef0d591a..7c1292637ed 100644 --- a/ui/desktop/src/App.tsx +++ b/ui/desktop/src/App.tsx @@ -20,11 +20,11 @@ import { createSession } from './sessions'; import { ChatType } from './types/chat'; import Hub from './components/Hub'; -import { ImageData } from './types/message'; +import { UserInput } from './types/message'; interface PairRouteState { resumeSessionId?: string; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; } import SettingsView, { SettingsViewOptions } from './components/settings/SettingsView'; import SessionsView from './components/sessions/SessionsView'; @@ -70,10 +70,10 @@ const PairRouteWrapper = ({ }: { activeSessions: Array<{ sessionId: string; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; }>; setActiveSessions: ( - sessions: Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }> + sessions: Array<{ sessionId: string; initialMessage?: UserInput }> ) => void; }) => { const { extensionsList } = useConfig(); @@ -359,7 +359,7 @@ export function AppInner() { const MAX_ACTIVE_SESSIONS = 10; const [activeSessions, setActiveSessions] = useState< - Array<{ sessionId: string; initialMessage?: { msg: string; images: ImageData[] } }> + Array<{ sessionId: string; initialMessage?: UserInput }> >([]); useEffect(() => { @@ -367,7 +367,7 @@ export function AppInner() { const { sessionId, initialMessage } = ( event as CustomEvent<{ sessionId: string; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; }> ).detail; diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 9e66db67ba7..cddc060c8db 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -28,7 +28,7 @@ import { useNavigation } from '../hooks/useNavigation'; import { RecipeHeader } from './RecipeHeader'; import { RecipeWarningModal } from './ui/RecipeWarningModal'; import { scanRecipe } from '../recipe'; -import { ImageData } from '../types/message'; +import { UserInput } from '../types/message'; import { useCostTracking } from '../hooks/useCostTracking'; import RecipeActivities from './recipes/RecipeActivities'; import { useToolCount } from './alerts/useToolCount'; @@ -57,7 +57,7 @@ interface BaseChatProps { suppressEmptyState: boolean; sessionId: string; isActiveSession: boolean; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; } export default function BaseChat({ diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index b7e0ce837e7..1219e41e172 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -586,6 +586,65 @@ export default function ChatInput({ })); }; + // Helper to convert images to ImageData format + const convertImagesToImageData = useCallback((): ImageData[] => { + const pastedImageData: ImageData[] = pastedImages + .filter((img) => img.dataUrl && !img.error && !img.isLoading) + .map((img) => { + const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], + mimeType: matches[1], + }; + } + return null; + }) + .filter((img): img is ImageData => img !== null); + + const droppedImageData: ImageData[] = allDroppedFiles + .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) + .map((file) => { + const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], + mimeType: matches[1], + }; + } + return null; + }) + .filter((img): img is ImageData => img !== null); + + return [...pastedImageData, ...droppedImageData]; + }, [pastedImages, allDroppedFiles]); + + // Helper to process dropped file paths and append to text + const appendDroppedFilePaths = useCallback((text: string): string => { + const droppedFilePaths = allDroppedFiles + .filter((file) => !file.isImage && !file.error && !file.isLoading) + .map((file) => file.path); + + if (droppedFilePaths.length > 0) { + const pathsString = droppedFilePaths.join(' '); + return text ? `${text} ${pathsString}` : pathsString; + } + return text; + }, [allDroppedFiles]); + + // Helper to clear input state after submission + const clearInputState = useCallback(() => { + setDisplayValue(''); + setValue(''); + setPastedImages([]); + if (onFilesProcessed && droppedFiles.length > 0) { + onFilesProcessed(); + } + if (localDroppedFiles.length > 0) { + setLocalDroppedFiles([]); + } + }, [droppedFiles.length, localDroppedFiles.length, onFilesProcessed, setLocalDroppedFiles]); + const compressImageDataUrl = async (dataUrl: string): Promise => { const res = await fetch(dataUrl); const blob = await res.blob(); @@ -778,45 +837,8 @@ export default function ChatInput({ return false; } - const pastedImageData: ImageData[] = pastedImages - .filter((img) => img.dataUrl && !img.error && !img.isLoading) - .map((img) => { - const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/); - if (matches) { - return { - data: matches[2], - mimeType: matches[1], - }; - } - return null; - }) - .filter((img): img is ImageData => img !== null); - - const droppedImageData: ImageData[] = allDroppedFiles - .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) - .map((file) => { - const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/); - if (matches) { - return { - data: matches[2], - mimeType: matches[1], - }; - } - return null; - }) - .filter((img): img is ImageData => img !== null); - - const imageData = [...pastedImageData, ...droppedImageData]; - - const droppedFilePaths = allDroppedFiles - .filter((file) => !file.isImage && !file.error && !file.isLoading) - .map((file) => file.path); - - let contentToQueue = displayValue.trim(); - if (droppedFilePaths.length > 0) { - const pathsString = droppedFilePaths.join(' '); - contentToQueue = contentToQueue ? `${contentToQueue} ${pathsString}` : pathsString; - } + const imageData = convertImagesToImageData(); + const contentToQueue = appendDroppedFilePaths(displayValue.trim()); const interruptionMatch = detectInterruption(displayValue.trim()); @@ -837,15 +859,7 @@ export default function ChatInput({ // Add the interruption message to the front of the queue so it gets sent first setQueuedMessages((prev) => [interruptionMessage, ...prev]); - setDisplayValue(''); - setValue(''); - setPastedImages([]); - if (onFilesProcessed && droppedFiles.length > 0) { - onFilesProcessed(); - } - if (localDroppedFiles.length > 0) { - setLocalDroppedFiles([]); - } + clearInputState(); return true; } @@ -864,15 +878,7 @@ export default function ChatInput({ } return newQueue; }); - setDisplayValue(''); - setValue(''); - setPastedImages([]); - if (onFilesProcessed && droppedFiles.length > 0) { - onFilesProcessed(); - } - if (localDroppedFiles.length > 0) { - setLocalDroppedFiles([]); - } + clearInputState(); return true; }; @@ -884,53 +890,20 @@ export default function ChatInput({ const performSubmit = useCallback( (text?: string) => { - const pastedImageData: ImageData[] = pastedImages - .filter((img) => img.dataUrl && !img.error && !img.isLoading) - .map((img) => { - const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/); - if (matches) { - return { - data: matches[2], // base64 data - mimeType: matches[1], // mime type - }; - } - return null; - }) - .filter((img): img is ImageData => img !== null); - - const droppedImageData: ImageData[] = allDroppedFiles - .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) - .map((file) => { - const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/); - if (matches) { - return { - data: matches[2], // base64 data - mimeType: matches[1], // mime type - }; - } - return null; - }) - .filter((img): img is ImageData => img !== null); - - const imageData = [...pastedImageData, ...droppedImageData]; - - const droppedFilePaths = allDroppedFiles - .filter((file) => !file.isImage && !file.error && !file.isLoading) - .map((file) => file.path); - - let textToSend = text ?? displayValue.trim(); - - // Add non-image dropped file paths to text - if (droppedFilePaths.length > 0) { - const pathsString = droppedFilePaths.join(' '); - textToSend = textToSend ? `${textToSend} ${pathsString}` : pathsString; - } + const imageData = convertImagesToImageData(); + const textToSend = appendDroppedFilePaths(text ?? displayValue.trim()); if (textToSend || imageData.length > 0) { + // Store original message in history if (displayValue.trim()) { LocalMessageStorage.addMessage(displayValue); - } else if (droppedFilePaths.length > 0) { - LocalMessageStorage.addMessage(droppedFilePaths.join(' ')); + } else { + const droppedFilePaths = allDroppedFiles + .filter((file) => !file.isImage && !file.error && !file.isLoading) + .map((file) => file.path); + if (droppedFilePaths.length > 0) { + LocalMessageStorage.addMessage(droppedFilePaths.join(' ')); + } } handleSubmit(textToSend, imageData); @@ -946,33 +919,21 @@ export default function ChatInput({ setLastInterruption(null); } - setDisplayValue(''); - setValue(''); - setPastedImages([]); + clearInputState(); setHistoryIndex(-1); setSavedInput(''); setIsInGlobalHistory(false); setHasUserTyped(false); - - // Clear both parent and local dropped files after processing - if (onFilesProcessed && droppedFiles.length > 0) { - onFilesProcessed(); - } - if (localDroppedFiles.length > 0) { - setLocalDroppedFiles([]); - } } }, [ - allDroppedFiles, + convertImagesToImageData, + appendDroppedFilePaths, displayValue, - droppedFiles.length, + allDroppedFiles, handleSubmit, lastInterruption, - localDroppedFiles.length, - onFilesProcessed, - pastedImages, - setLocalDroppedFiles, + clearInputState, ] ); @@ -1250,14 +1211,7 @@ export default function ChatInput({ onDrop={handleLocalDrop} onDragOver={handleLocalDragOver} > - {/* Hidden file input */} - + {/* Message Queue Display */} {queuedMessages.length > 0 && ( void; activeSessions: Array<{ sessionId: string; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; }>; } diff --git a/ui/desktop/src/components/Layout/AppLayout.tsx b/ui/desktop/src/components/Layout/AppLayout.tsx index c895d6c0df5..99f72136141 100644 --- a/ui/desktop/src/components/Layout/AppLayout.tsx +++ b/ui/desktop/src/components/Layout/AppLayout.tsx @@ -7,12 +7,12 @@ import { Button } from '../ui/button'; import { Sidebar, SidebarInset, SidebarProvider, SidebarTrigger, useSidebar } from '../ui/sidebar'; import ChatSessionsContainer from '../ChatSessionsContainer'; import { useChatContext } from '../../contexts/ChatContext'; -import { ImageData } from '../../types/message'; +import { UserInput } from '../../types/message'; interface AppLayoutContentProps { activeSessions: Array<{ sessionId: string; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; }>; } @@ -132,7 +132,7 @@ const AppLayoutContent: React.FC = ({ activeSessions }) = interface AppLayoutProps { activeSessions: Array<{ sessionId: string; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; }>; } diff --git a/ui/desktop/src/hooks/useAutoSubmit.ts b/ui/desktop/src/hooks/useAutoSubmit.ts index 837d6e96058..11962431464 100644 --- a/ui/desktop/src/hooks/useAutoSubmit.ts +++ b/ui/desktop/src/hooks/useAutoSubmit.ts @@ -4,7 +4,7 @@ import { useSearchParams } from 'react-router-dom'; import { Session } from '../api'; import { Message } from '../api'; import { ChatState } from '../types/chatState'; -import { ImageData } from '../types/message'; +import { UserInput, ImageData } from '../types/message'; /** * Auto-submit scenarios: @@ -18,7 +18,7 @@ interface UseAutoSubmitProps { session: Session | undefined; messages: Message[]; chatState: ChatState; - initialMessage: { msg: string; images: ImageData[] } | undefined; + initialMessage: UserInput | undefined; handleSubmit: (message: string, images: ImageData[]) => void; } diff --git a/ui/desktop/src/hooks/useFileDrop.ts b/ui/desktop/src/hooks/useFileDrop.ts index bd4c5d19218..226a5c94ab7 100644 --- a/ui/desktop/src/hooks/useFileDrop.ts +++ b/ui/desktop/src/hooks/useFileDrop.ts @@ -6,7 +6,7 @@ export interface DroppedFile { name: string; type: string; isImage: boolean; - dataUrl?: string; // For images: compressed base64 data URL for direct inclusion + dataUrl?: string; isLoading?: boolean; error?: string; } @@ -109,7 +109,6 @@ export const useFileDrop = () => { droppedFileObjects.push(droppedFile); - // For images, read and compress for direct inclusion if (droppedFile.isImage && !droppedFile.error) { const reader = new FileReader(); activeReadersRef.current.add(reader); diff --git a/ui/desktop/src/types/message.ts b/ui/desktop/src/types/message.ts index 9539ca7fe55..83964a65e7b 100644 --- a/ui/desktop/src/types/message.ts +++ b/ui/desktop/src/types/message.ts @@ -12,6 +12,11 @@ export interface ImageData { mimeType: string; } +export interface UserInput { + msg: string; + images: ImageData[]; +} + export function createUserMessage(text: string, images?: ImageData[]): Message { const content: Message['content'] = []; diff --git a/ui/desktop/src/utils/navigationUtils.ts b/ui/desktop/src/utils/navigationUtils.ts index e2289e536e8..33f29f068ef 100644 --- a/ui/desktop/src/utils/navigationUtils.ts +++ b/ui/desktop/src/utils/navigationUtils.ts @@ -1,6 +1,6 @@ import { NavigateFunction } from 'react-router-dom'; import { Recipe } from '../api'; -import { ImageData } from '../types/message'; +import { UserInput } from '../types/message'; export type View = | 'welcome' @@ -30,7 +30,7 @@ export type ViewOptions = { parentView?: View; parentViewOptions?: ViewOptions; disableAnimation?: boolean; - initialMessage?: { msg: string; images: ImageData[] }; + initialMessage?: UserInput; shareToken?: string; resumeSessionId?: string; pendingScheduleDeepLink?: string; From 1c9ed327082bc585e86d809d8b0cdd9044617745 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 16:15:17 -0500 Subject: [PATCH 09/13] Centralize --- ui/desktop/src/components/ChatInput.tsx | 19 +----------- ui/desktop/src/hooks/useFileDrop.ts | 39 +------------------------ ui/desktop/src/types/message.ts | 2 -- ui/desktop/src/utils/conversionUtils.ts | 18 ++++++++++++ 4 files changed, 20 insertions(+), 58 deletions(-) diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index 1219e41e172..3db2bd4b11f 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -42,6 +42,7 @@ import { } from '../utils/analytics'; import { getNavigationShortcutText } from '../utils/keyboardShortcuts'; import { ImageData } from '../types/message'; +import { compressImageDataUrl } from '../utils/conversionUtils'; interface PastedImage { id: string; @@ -645,24 +646,6 @@ export default function ChatInput({ } }, [droppedFiles.length, localDroppedFiles.length, onFilesProcessed, setLocalDroppedFiles]); - const compressImageDataUrl = async (dataUrl: string): Promise => { - const res = await fetch(dataUrl); - const blob = await res.blob(); - const bitmap = await globalThis.createImageBitmap(blob); - - const maxDim = 1024; - const scale = Math.min(1, maxDim / Math.max(bitmap.width, bitmap.height)); - const width = Math.floor(bitmap.width * scale); - const height = Math.floor(bitmap.height * scale); - - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - canvas.getContext('2d')!.drawImage(bitmap, 0, 0, width, height); - - return canvas.toDataURL('image/jpeg', 0.85); - }; - const handlePaste = async (evt: React.ClipboardEvent) => { const files = Array.from(evt.clipboardData.files || []); const imageFiles = files.filter((file) => file.type.startsWith('image/')); diff --git a/ui/desktop/src/hooks/useFileDrop.ts b/ui/desktop/src/hooks/useFileDrop.ts index 226a5c94ab7..40de5e012b1 100644 --- a/ui/desktop/src/hooks/useFileDrop.ts +++ b/ui/desktop/src/hooks/useFileDrop.ts @@ -1,4 +1,5 @@ import { useCallback, useState, useRef, useEffect } from 'react'; +import { compressImageDataUrl } from '../utils/conversionUtils'; export interface DroppedFile { id: string; @@ -11,44 +12,6 @@ export interface DroppedFile { error?: string; } -// Helper function to compress image data URLs -const compressImageDataUrl = async (dataUrl: string): Promise => { - return new Promise((resolve, reject) => { - const img = new globalThis.Image(); - img.onload = () => { - const canvas = document.createElement('canvas'); - const ctx = canvas.getContext('2d'); - if (!ctx) { - reject(new Error('Failed to get canvas context')); - return; - } - - // Resize to max 1024px on longest side - const maxSize = 1024; - let width = img.width; - let height = img.height; - - if (width > height && width > maxSize) { - height = (height * maxSize) / width; - width = maxSize; - } else if (height > maxSize) { - width = (width * maxSize) / height; - height = maxSize; - } - - canvas.width = width; - canvas.height = height; - ctx.drawImage(img, 0, 0, width, height); - - // Convert to JPEG with 0.85 quality - const compressedDataUrl = canvas.toDataURL('image/jpeg', 0.85); - resolve(compressedDataUrl); - }; - img.onerror = () => reject(new Error('Failed to load image')); - img.src = dataUrl; - }); -}; - export const useFileDrop = () => { const [droppedFiles, setDroppedFiles] = useState([]); const activeReadersRef = useRef>(new Set()); diff --git a/ui/desktop/src/types/message.ts b/ui/desktop/src/types/message.ts index 83964a65e7b..43423578ede 100644 --- a/ui/desktop/src/types/message.ts +++ b/ui/desktop/src/types/message.ts @@ -20,12 +20,10 @@ export interface UserInput { export function createUserMessage(text: string, images?: ImageData[]): Message { const content: Message['content'] = []; - // Add text content if present if (text.trim()) { content.push({ type: 'text', text }); } - // Add image content if present if (images && images.length > 0) { images.forEach((img) => { content.push({ diff --git a/ui/desktop/src/utils/conversionUtils.ts b/ui/desktop/src/utils/conversionUtils.ts index 51b9cb30acc..3f6c30ce91e 100644 --- a/ui/desktop/src/utils/conversionUtils.ts +++ b/ui/desktop/src/utils/conversionUtils.ts @@ -21,3 +21,21 @@ export function errorMessage(err: Error | unknown, default_value?: string) { return default_value || String(err); } } + +export async function compressImageDataUrl(dataUrl: string): Promise { + const res = await fetch(dataUrl); + const blob = await res.blob(); + const bitmap = await globalThis.createImageBitmap(blob); + + const maxDim = 1024; + const scale = Math.min(1, maxDim / Math.max(bitmap.width, bitmap.height)); + const width = Math.floor(bitmap.width * scale); + const height = Math.floor(bitmap.height * scale); + + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + canvas.getContext('2d')!.drawImage(bitmap, 0, 0, width, height); + + return canvas.toDataURL('image/jpeg', 0.85); +} From 488c36ccf3ce6c17e708c27a40175842cf39a4b7 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 16:18:35 -0500 Subject: [PATCH 10/13] Skip empty text --- ui/desktop/src/components/GooseMessage.tsx | 10 ++++++---- ui/desktop/src/components/UserMessage.tsx | 16 +++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ui/desktop/src/components/GooseMessage.tsx b/ui/desktop/src/components/GooseMessage.tsx index c6b593f50b0..16de3de51cb 100644 --- a/ui/desktop/src/components/GooseMessage.tsx +++ b/ui/desktop/src/components/GooseMessage.tsx @@ -112,11 +112,13 @@ export default function GooseMessage({ )} - {displayText && ( + {(displayText.trim() || imagePaths.length > 0) && (
-
- -
+ {displayText.trim() && ( +
+ +
+ )} {imagePaths.length > 0 && (
diff --git a/ui/desktop/src/components/UserMessage.tsx b/ui/desktop/src/components/UserMessage.tsx index b691a6ed0a3..a022fd8f17b 100644 --- a/ui/desktop/src/components/UserMessage.tsx +++ b/ui/desktop/src/components/UserMessage.tsx @@ -194,14 +194,16 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
-
-
- + {textContent.trim() && ( +
+
+ +
-
+ )} {imagePaths.length > 0 && (
From 5409548c63e18c6a518d4577216a688a9059f303 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 16:23:59 -0500 Subject: [PATCH 11/13] use UserInput more --- ui/desktop/src/components/BaseChat.tsx | 12 ++++++------ ui/desktop/src/components/ChatInput.tsx | 19 +++++++------------ ui/desktop/src/components/Hub.tsx | 5 +++-- ui/desktop/src/hooks/useAutoSubmit.ts | 10 +++++----- ui/desktop/src/hooks/useChatStream.ts | 9 +++++---- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index cddc060c8db..bdc7fa765f6 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -160,11 +160,11 @@ export default function BaseChat({ .reverse(); }, [messages]); - const chatInputSubmit = (textValue: string, images: ImageData[]) => { - if (recipe && textValue.trim()) { + const chatInputSubmit = (input: UserInput) => { + if (recipe && input.msg.trim()) { setHasStartedUsingRecipe(true); } - handleSubmit(textValue, images); + handleSubmit(input); }; const { sessionCosts } = useCostTracking({ @@ -403,7 +403,7 @@ export default function BaseChat({ {recipe && (
handleSubmit(text, [])} + append={(text: string) => handleSubmit({ msg: text, images: [] })} activities={Array.isArray(recipe.activities) ? recipe.activities : null} title={recipe.title} parameterValues={session?.user_recipe_values || {}} @@ -418,7 +418,7 @@ export default function BaseChat({ messages={messages} chat={{ sessionId }} toolCallNotifications={toolCallNotifications} - append={(text: string) => handleSubmit(text, [])} + append={(text: string) => handleSubmit({ msg: text, images: [] })} isUserMessage={(m: Message) => m.role === 'user'} isStreamingMessage={chatState !== ChatState.Idle} onRenderingComplete={handleRenderingComplete} @@ -430,7 +430,7 @@ export default function BaseChat({
) : !recipe && showPopularTopics ? ( - handleSubmit(text, [])} /> + handleSubmit({ msg: text, images: [] })} /> ) : null} diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index 3db2bd4b11f..2d77daae851 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -41,7 +41,7 @@ import { trackEditRecipeOpened, } from '../utils/analytics'; import { getNavigationShortcutText } from '../utils/keyboardShortcuts'; -import { ImageData } from '../types/message'; +import { UserInput, ImageData } from '../types/message'; import { compressImageDataUrl } from '../utils/conversionUtils'; interface PastedImage { @@ -67,7 +67,7 @@ interface ModelLimit { interface ChatInputProps { sessionId: string | null; - handleSubmit: (userMessage: string, images: ImageData[]) => void; + handleSubmit: (input: UserInput) => void; chatState: ChatState; setChatState?: (state: ChatState) => void; onStop?: () => void; @@ -212,7 +212,7 @@ export default function ChatInput({ if (shouldProcessQueue) { const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); - handleSubmit(nextMessage.content, nextMessage.images); + handleSubmit({ msg: nextMessage.content, images: nextMessage.images }); setQueuedMessages((prev) => { const newQueue = prev.slice(1); // If queue becomes empty after processing, clear the paused state @@ -370,7 +370,6 @@ export default function ChatInput({ return []; }; - // Helper function to find model limit using pattern matching const findModelLimit = (modelName: string, modelLimits: ModelLimit[]): number | null => { if (!modelName) return null; const matchingLimit = modelLimits.find((limit) => @@ -459,7 +458,7 @@ export default function ChatInput({ compactButtonDisabled: !totalTokens, onCompact: () => { window.dispatchEvent(new CustomEvent(AppEvents.HIDE_ALERT_POPOVER)); - handleSubmit(MANUAL_COMPACT_TRIGGER, []); + handleSubmit({ msg: MANUAL_COMPACT_TRIGGER, images: [] }); }, compactIcon: , }); @@ -587,7 +586,6 @@ export default function ChatInput({ })); }; - // Helper to convert images to ImageData format const convertImagesToImageData = useCallback((): ImageData[] => { const pastedImageData: ImageData[] = pastedImages .filter((img) => img.dataUrl && !img.error && !img.isLoading) @@ -620,7 +618,6 @@ export default function ChatInput({ return [...pastedImageData, ...droppedImageData]; }, [pastedImages, allDroppedFiles]); - // Helper to process dropped file paths and append to text const appendDroppedFilePaths = useCallback((text: string): string => { const droppedFilePaths = allDroppedFiles .filter((file) => !file.isImage && !file.error && !file.isLoading) @@ -633,7 +630,6 @@ export default function ChatInput({ return text; }, [allDroppedFiles]); - // Helper to clear input state after submission const clearInputState = useCallback(() => { setDisplayValue(''); setValue(''); @@ -814,7 +810,6 @@ export default function ChatInput({ } }; - // Helper function to handle interruption and queue logic when loading const handleInterruptionAndQueue = () => { if (!isLoading || !hasSubmittableContent) { return false; @@ -889,7 +884,7 @@ export default function ChatInput({ } } - handleSubmit(textToSend, imageData); + handleSubmit({ msg: textToSend, images: imageData }); // Auto-resume queue after sending a NON-interruption message (if it was paused due to interruption) if ( @@ -1154,7 +1149,7 @@ export default function ChatInput({ // Remove the message from queue and send it immediately setQueuedMessages((prev) => prev.filter((msg) => msg.id !== messageId)); LocalMessageStorage.addMessage(messageToSend.content); - handleSubmit(messageToSend.content, messageToSend.images); + handleSubmit({ msg: messageToSend.content, images: messageToSend.images }); // Restore previous pause state after a brief delay to prevent race condition setTimeout(() => { @@ -1168,7 +1163,7 @@ export default function ChatInput({ if (!isLoading && queuedMessages.length > 0) { const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); - handleSubmit(nextMessage.content, nextMessage.images); + handleSubmit({ msg: nextMessage.content, images: nextMessage.images }); setQueuedMessages((prev) => { const newQueue = prev.slice(1); // If queue becomes empty after processing, clear the paused state diff --git a/ui/desktop/src/components/Hub.tsx b/ui/desktop/src/components/Hub.tsx index 583e9908f0f..a36cff26fc6 100644 --- a/ui/desktop/src/components/Hub.tsx +++ b/ui/desktop/src/components/Hub.tsx @@ -29,7 +29,7 @@ import { import { getInitialWorkingDir } from '../utils/workingDir'; import { createSession } from '../sessions'; import LoadingGoose from './LoadingGoose'; -import { ImageData } from '../types/message'; +import { UserInput } from '../types/message'; export default function Hub({ setView, @@ -40,7 +40,8 @@ export default function Hub({ const [workingDir, setWorkingDir] = useState(getInitialWorkingDir()); const [isCreatingSession, setIsCreatingSession] = useState(false); - const handleSubmit = async (userMessage: string, images: ImageData[]) => { + const handleSubmit = async (input: UserInput) => { + const { msg: userMessage, images } = input; if ((images.length > 0 || userMessage.trim()) && !isCreatingSession) { const extensionConfigs = getExtensionConfigsWithOverrides(extensionsList); clearExtensionOverrides(); diff --git a/ui/desktop/src/hooks/useAutoSubmit.ts b/ui/desktop/src/hooks/useAutoSubmit.ts index 11962431464..5380a9cf48d 100644 --- a/ui/desktop/src/hooks/useAutoSubmit.ts +++ b/ui/desktop/src/hooks/useAutoSubmit.ts @@ -4,7 +4,7 @@ import { useSearchParams } from 'react-router-dom'; import { Session } from '../api'; import { Message } from '../api'; import { ChatState } from '../types/chatState'; -import { UserInput, ImageData } from '../types/message'; +import { UserInput } from '../types/message'; /** * Auto-submit scenarios: @@ -19,7 +19,7 @@ interface UseAutoSubmitProps { messages: Message[]; chatState: ChatState; initialMessage: UserInput | undefined; - handleSubmit: (message: string, images: ImageData[]) => void; + handleSubmit: (input: UserInput) => void; } interface UseAutoSubmitReturn { @@ -69,7 +69,7 @@ export function useAutoSubmit({ // Hub always creates new sessions, so message_count will be 0 if (initialMessage && session.message_count === 0 && messages.length === 0) { hasAutoSubmittedRef.current = true; - handleSubmit(initialMessage.msg, initialMessage.images); + handleSubmit(initialMessage); clearInitialMessage(); return; } @@ -77,7 +77,7 @@ export function useAutoSubmit({ // Scenario 2: Forked session with edited message if (shouldStartAgent && initialMessage) { hasAutoSubmittedRef.current = true; - handleSubmit(initialMessage.msg, initialMessage.images); + handleSubmit(initialMessage); clearInitialMessage(); return; } @@ -85,7 +85,7 @@ export function useAutoSubmit({ // Scenario 3: Resume with shouldStartAgent (continue existing conversation) if (shouldStartAgent) { hasAutoSubmittedRef.current = true; - handleSubmit('', []); + handleSubmit({ msg: '', images: [] }); } }, [ session, diff --git a/ui/desktop/src/hooks/useChatStream.ts b/ui/desktop/src/hooks/useChatStream.ts index 0e0a10c7a75..0de7624d707 100644 --- a/ui/desktop/src/hooks/useChatStream.ts +++ b/ui/desktop/src/hooks/useChatStream.ts @@ -21,7 +21,7 @@ import { getCompactingMessage, getThinkingMessage, NotificationEvent, - ImageData, + UserInput, } from '../types/message'; import { errorMessage } from '../utils/conversionUtils'; import { showExtensionLoadResults } from '../utils/extensionErrorUtils'; @@ -39,7 +39,7 @@ interface UseChatStreamReturn { messages: Message[]; chatState: ChatState; setChatState: (state: ChatState) => void; - handleSubmit: (userMessage: string, images: ImageData[]) => Promise; + handleSubmit: (input: UserInput) => Promise; submitElicitationResponse: ( elicitationId: string, userData: Record @@ -447,7 +447,8 @@ export function useChatStream({ }, [sessionId, onSessionLoaded]); const handleSubmit = useCallback( - async (userMessage: string, images: ImageData[]) => { + async (input: UserInput) => { + const { msg: userMessage, images } = input; const currentState = stateRef.current; // Guard: Don't submit if session hasn't been loaded yet @@ -694,7 +695,7 @@ export function useChatStream({ if (sessionResponse.data?.conversation) { dispatch({ type: 'SET_MESSAGES', payload: sessionResponse.data.conversation }); } - await handleSubmit(newContent, []); + await handleSubmit({ msg: newContent, images: [] }); } } catch (error) { const errorMsg = errorMessage(error); From 0d5f36bdb9cacc549a3d46262497dd5b573a7074 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 16:37:45 -0500 Subject: [PATCH 12/13] Too clever for our own good! --- ui/desktop/src/utils/conversionUtils.ts | 34 +++++++++++++++---------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/ui/desktop/src/utils/conversionUtils.ts b/ui/desktop/src/utils/conversionUtils.ts index 3f6c30ce91e..79f512f7ae4 100644 --- a/ui/desktop/src/utils/conversionUtils.ts +++ b/ui/desktop/src/utils/conversionUtils.ts @@ -23,19 +23,27 @@ export function errorMessage(err: Error | unknown, default_value?: string) { } export async function compressImageDataUrl(dataUrl: string): Promise { - const res = await fetch(dataUrl); - const blob = await res.blob(); - const bitmap = await globalThis.createImageBitmap(blob); + return new Promise((resolve, reject) => { + const img = new globalThis.Image(); + img.onload = () => { + const maxDim = 1024; + const scale = Math.min(1, maxDim / Math.max(img.width, img.height)); + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); - const maxDim = 1024; - const scale = Math.min(1, maxDim / Math.max(bitmap.width, bitmap.height)); - const width = Math.floor(bitmap.width * scale); - const height = Math.floor(bitmap.height * scale); + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (!ctx) { + reject(new Error('Failed to get canvas context')); + return; + } + ctx.drawImage(img, 0, 0, width, height); - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - canvas.getContext('2d')!.drawImage(bitmap, 0, 0, width, height); - - return canvas.toDataURL('image/jpeg', 0.85); + resolve(canvas.toDataURL('image/jpeg', 0.85)); + }; + img.onerror = () => reject(new Error('Failed to load image')); + img.src = dataUrl; + }); } From 027e9c59d754cf3184f3c363ef08c916fa0658a5 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Thu, 22 Jan 2026 17:58:16 -0500 Subject: [PATCH 13/13] marker --- ui/desktop/src/utils/conversionUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/desktop/src/utils/conversionUtils.ts b/ui/desktop/src/utils/conversionUtils.ts index 3bffb0aa2f0..4e6a2fce8a5 100644 --- a/ui/desktop/src/utils/conversionUtils.ts +++ b/ui/desktop/src/utils/conversionUtils.ts @@ -46,6 +46,7 @@ export async function compressImageDataUrl(dataUrl: string): Promise { img.onerror = () => reject(new Error('Failed to load image')); img.src = dataUrl; }); +} export function formatAppName(name: string): string { return name @@ -53,5 +54,4 @@ export function formatAppName(name: string): string { .filter((word) => word.length > 0) .map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) .join(' '); ->>>>>>> origin/main }