diff --git a/ui/desktop/src/App.tsx b/ui/desktop/src/App.tsx index 84dea4be434a..161cbcea02ce 100644 --- a/ui/desktop/src/App.tsx +++ b/ui/desktop/src/App.tsx @@ -20,10 +20,11 @@ import { createSession } from './sessions'; import { ChatType } from './types/chat'; import Hub from './components/Hub'; +import { UserInput } from './types/message'; interface PairRouteState { resumeSessionId?: string; - initialMessage?: string; + initialMessage?: UserInput; } import SettingsView, { SettingsViewOptions } from './components/settings/SettingsView'; import SessionsView from './components/sessions/SessionsView'; @@ -68,8 +69,13 @@ const HubRouteWrapper = () => { const PairRouteWrapper = ({ activeSessions, }: { - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; - setActiveSessions: (sessions: Array<{ sessionId: string; initialMessage?: string }>) => void; + activeSessions: Array<{ + sessionId: string; + initialMessage?: UserInput; + }>; + setActiveSessions: ( + sessions: Array<{ sessionId: string; initialMessage?: UserInput }> + ) => void; }) => { const { extensionsList } = useConfig(); const location = useLocation(); @@ -354,13 +360,16 @@ export function AppInner() { const MAX_ACTIVE_SESSIONS = 10; const [activeSessions, setActiveSessions] = useState< - Array<{ sessionId: string; initialMessage?: string }> + Array<{ sessionId: string; initialMessage?: UserInput }> >([]); useEffect(() => { const handleAddActiveSession = (event: Event) => { const { sessionId, initialMessage } = ( - event as CustomEvent<{ sessionId: string; initialMessage?: string }> + event as CustomEvent<{ + sessionId: string; + initialMessage?: UserInput; + }> ).detail; setActiveSessions((prev) => { diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 601d1a030727..44d6a2c3e893 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -28,17 +28,18 @@ import { useNavigation } from '../hooks/useNavigation'; import { RecipeHeader } from './RecipeHeader'; import { RecipeWarningModal } from './ui/RecipeWarningModal'; import { scanRecipe } from '../recipe'; +import { UserInput } from '../types/message'; import { useCostTracking } from '../hooks/useCostTracking'; import RecipeActivities from './recipes/RecipeActivities'; import { useToolCount } from './alerts/useToolCount'; -import { getThinkingMessage, getTextContent } from '../types/message'; +import { getThinkingMessage, getTextAndImageContent } from '../types/message'; import ParameterInputModal from './ParameterInputModal'; import { substituteParameters } from '../utils/providerUtils'; import CreateRecipeFromSessionModal from './recipes/CreateRecipeFromSessionModal'; import { toastSuccess } from '../toasts'; import { Recipe } from '../recipe'; import { useAutoSubmit } from '../hooks/useAutoSubmit'; -import { Goose } from './icons/Goose'; +import { Goose } from './icons'; import EnvironmentBadge from './GooseSidebar/EnvironmentBadge'; const CurrentModelContext = createContext<{ model: string; mode: string } | null>(null); @@ -56,10 +57,10 @@ interface BaseChatProps { suppressEmptyState: boolean; sessionId: string; isActiveSession: boolean; - initialMessage?: string; + initialMessage?: UserInput; } -function BaseChatContent({ +export default function BaseChat({ setChat, renderHeader, customChatInputProps = {}, @@ -149,7 +150,7 @@ function BaseChatContent({ return messages .reduce((history, message) => { if (message.role === 'user') { - const text = getTextContent(message).trim(); + const text = getTextAndImageContent(message).textContent.trim(); if (text) { history.push(text); } @@ -159,14 +160,11 @@ function BaseChatContent({ .reverse(); }, [messages]); - const handleFormSubmit = (e: React.FormEvent) => { - const customEvent = e as unknown as CustomEvent; - const textValue = customEvent.detail?.value || ''; - - if (recipe && textValue.trim()) { + const chatInputSubmit = (input: UserInput) => { + if (recipe && input.msg.trim()) { setHasStartedUsingRecipe(true); } - handleSubmit(textValue); + handleSubmit(input); }; const { sessionCosts } = useCostTracking({ @@ -326,7 +324,6 @@ function BaseChatContent({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [session?.name, setChat]); - // Only use initialMessage for the prompt if it hasn't been submitted yet // If we have a recipe prompt and user recipe values, substitute parameters let recipePrompt = ''; if (messages.length === 0 && recipe?.prompt) { @@ -413,7 +410,7 @@ function BaseChatContent({ {recipe && (
handleSubmit(text)} + append={(text: string) => handleSubmit({ msg: text, images: [] })} activities={Array.isArray(recipe.activities) ? recipe.activities : null} title={recipe.title} parameterValues={session?.user_recipe_values || {}} @@ -428,7 +425,7 @@ function BaseChatContent({ messages={messages} chat={{ sessionId }} toolCallNotifications={toolCallNotifications} - append={(text: string) => handleSubmit(text)} + append={(text: string) => handleSubmit({ msg: text, images: [] })} isUserMessage={(m: Message) => m.role === 'user'} isStreamingMessage={chatState !== ChatState.Idle} onRenderingComplete={handleRenderingComplete} @@ -440,7 +437,7 @@ function BaseChatContent({
) : !recipe && showPopularTopics ? ( - handleSubmit(text)} /> + handleSubmit({ msg: text, images: [] })} /> ) : null} @@ -464,7 +461,7 @@ function BaseChatContent({ ); } - -export default function BaseChat(props: BaseChatProps) { - return ; -} diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index 659d46940010..a53df80e474d 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -25,7 +25,7 @@ import { COST_TRACKING_ENABLED, VOICE_DICTATION_ELEVENLABS_ENABLED } from '../up import { CostTracker } from './bottom_menu/CostTracker'; import { DroppedFile, useFileDrop } from '../hooks/useFileDrop'; import { Recipe } from '../recipe'; -import MessageQueue from './MessageQueue'; +import { MessageQueue, QueuedMessage } from './MessageQueue'; import { detectInterruption } from '../utils/interruptionDetector'; import { DiagnosticsModal } from './ui/Diagnostics'; import { getSession, Message } from '../api'; @@ -41,24 +41,17 @@ import { trackEditRecipeOpened, } from '../utils/analytics'; import { getNavigationShortcutText } from '../utils/keyboardShortcuts'; - -interface QueuedMessage { - id: string; - content: string; - timestamp: number; -} +import { UserInput, ImageData } from '../types/message'; +import { compressImageDataUrl } from '../utils/conversionUtils'; interface PastedImage { id: string; - dataUrl: string; // For immediate preview - filePath?: string; // Path on filesystem after saving + dataUrl: string; isLoading: boolean; error?: string; } -// Constants for image handling -const MAX_IMAGES_PER_MESSAGE = 5; -const MAX_IMAGE_SIZE_MB = 5; +const MAX_IMAGES_PER_MESSAGE = 10; // Constants for token and tool alerts const TOKEN_LIMIT_DEFAULT = 128000; // fallback for custom models that the backend doesn't know about @@ -74,7 +67,7 @@ interface ModelLimit { interface ChatInputProps { sessionId: string | null; - handleSubmit: (e: React.FormEvent) => void; + handleSubmit: (input: UserInput) => void; chatState: ChatState; setChatState?: (state: ChatState) => void; onStop?: () => void; @@ -219,11 +212,7 @@ export default function ChatInput({ if (shouldProcessQueue) { const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); - handleSubmit( - new CustomEvent('submit', { - detail: { value: nextMessage.content }, - }) as unknown as React.FormEvent - ); + handleSubmit({ msg: nextMessage.content, images: nextMessage.images }); setQueuedMessages((prev) => { const newQueue = prev.slice(1); // If queue becomes empty after processing, clear the paused state @@ -309,15 +298,7 @@ export default function ChatInput({ useEffect(() => { setValue(initialValue); setDisplayValue(initialValue); - setPastedImages((currentPastedImages) => { - currentPastedImages.forEach((img) => { - if (img.filePath) { - window.electron.deleteTempFile(img.filePath); - } - }); - return []; - }); - + setPastedImages([]); setHistoryIndex(-1); setIsInGlobalHistory(false); setHasUserTyped(false); @@ -366,43 +347,9 @@ export default function ChatInput({ }; const handleRemovePastedImage = (idToRemove: string) => { - const imageToRemove = pastedImages.find((img) => img.id === idToRemove); - if (imageToRemove?.filePath) { - window.electron.deleteTempFile(imageToRemove.filePath); - } setPastedImages((currentImages) => currentImages.filter((img) => img.id !== idToRemove)); }; - const handleRetryImageSave = async (imageId: string) => { - const imageToRetry = pastedImages.find((img) => img.id === imageId); - if (!imageToRetry || !imageToRetry.dataUrl) return; - - // Set the image to loading state - setPastedImages((prev) => - prev.map((img) => (img.id === imageId ? { ...img, isLoading: true, error: undefined } : img)) - ); - - try { - const result = await window.electron.saveDataUrlToTemp(imageToRetry.dataUrl, imageId); - setPastedImages((prev) => - prev.map((img) => - img.id === result.id - ? { ...img, filePath: result.filePath, error: result.error, isLoading: false } - : img - ) - ); - } catch (err) { - console.error('Error retrying image save:', err); - setPastedImages((prev) => - prev.map((img) => - img.id === imageId - ? { ...img, error: 'Failed to save image via Electron.', isLoading: false } - : img - ) - ); - } - }; - useEffect(() => { if (textAreaRef.current) { textAreaRef.current.focus(); @@ -423,7 +370,6 @@ export default function ChatInput({ return []; }; - // Helper function to find model limit using pattern matching const findModelLimit = (modelName: string, modelLimits: ModelLimit[]): number | null => { if (!modelName) return null; const matchingLimit = modelLimits.find((limit) => @@ -512,12 +458,7 @@ export default function ChatInput({ compactButtonDisabled: !totalTokens, onCompact: () => { window.dispatchEvent(new CustomEvent(AppEvents.HIDE_ALERT_POPOVER)); - - const customEvent = new CustomEvent('submit', { - detail: { value: MANUAL_COMPACT_TRIGGER }, - }) as unknown as React.FormEvent; - - handleSubmit(customEvent); + handleSubmit({ msg: MANUAL_COMPACT_TRIGGER, images: [] }); }, compactIcon: , }); @@ -542,20 +483,6 @@ export default function ChatInput({ // Cleanup effect for component unmount - prevent memory leaks useEffect(() => { return () => { - // Clear any pending timeouts from image processing - setPastedImages((currentImages) => { - currentImages.forEach((img) => { - if (img.filePath) { - try { - window.electron.deleteTempFile(img.filePath); - } catch (error) { - console.error('Error deleting temp file:', error); - } - } - }); - return []; - }); - // Clear all tracked timeouts // eslint-disable-next-line react-hooks/exhaustive-deps const timeouts = timeoutRefsRef.current; @@ -659,6 +586,62 @@ export default function ChatInput({ })); }; + const convertImagesToImageData = useCallback((): ImageData[] => { + const pastedImageData: ImageData[] = pastedImages + .filter((img) => img.dataUrl && !img.error && !img.isLoading) + .map((img) => { + const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], + mimeType: matches[1], + }; + } + return null; + }) + .filter((img): img is ImageData => img !== null); + + const droppedImageData: ImageData[] = allDroppedFiles + .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading) + .map((file) => { + const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + return { + data: matches[2], + mimeType: matches[1], + }; + } + return null; + }) + .filter((img): img is ImageData => img !== null); + + return [...pastedImageData, ...droppedImageData]; + }, [pastedImages, allDroppedFiles]); + + const appendDroppedFilePaths = useCallback((text: string): string => { + const droppedFilePaths = allDroppedFiles + .filter((file) => !file.isImage && !file.error && !file.isLoading) + .map((file) => file.path); + + if (droppedFilePaths.length > 0) { + const pathsString = droppedFilePaths.join(' '); + return text ? `${text} ${pathsString}` : pathsString; + } + return text; + }, [allDroppedFiles]); + + const clearInputState = useCallback(() => { + setDisplayValue(''); + setValue(''); + setPastedImages([]); + if (onFilesProcessed && droppedFiles.length > 0) { + onFilesProcessed(); + } + if (localDroppedFiles.length > 0) { + setLocalDroppedFiles([]); + } + }, [droppedFiles.length, localDroppedFiles.length, onFilesProcessed, setLocalDroppedFiles]); + const handlePaste = async (evt: React.ClipboardEvent) => { const files = Array.from(evt.clipboardData.files || []); const imageFiles = files.filter((file) => file.type.startsWith('image/')); @@ -694,26 +677,6 @@ export default function ChatInput({ const newImages: PastedImage[] = []; for (const file of imageFiles) { - // Check individual file size before processing - if (file.size > MAX_IMAGE_SIZE_MB * 1024 * 1024) { - const errorId = `error-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; - newImages.push({ - id: errorId, - dataUrl: '', - isLoading: false, - error: `Image too large (${Math.round(file.size / (1024 * 1024))}MB). Maximum ${MAX_IMAGE_SIZE_MB}MB allowed.`, - }); - - // Remove the error message after 5 seconds with cleanup tracking - const timeoutId = setTimeout(() => { - setPastedImages((prev) => prev.filter((img) => img.id !== errorId)); - timeoutRefsRef.current.delete(timeoutId); - }, 5000); - timeoutRefsRef.current.add(timeoutId); - - continue; - } - const imageId = `img-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; // Add the image with loading state @@ -728,30 +691,12 @@ export default function ChatInput({ reader.onload = async (e) => { const dataUrl = e.target?.result as string; if (dataUrl) { - // Update the image with the data URL + const compressedDataUrl = await compressImageDataUrl(dataUrl); setPastedImages((prev) => - prev.map((img) => (img.id === imageId ? { ...img, dataUrl, isLoading: true } : img)) + prev.map((img) => + img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: false } : img + ) ); - - try { - const result = await window.electron.saveDataUrlToTemp(dataUrl, imageId); - setPastedImages((prev) => - prev.map((img) => - img.id === result.id - ? { ...img, filePath: result.filePath, error: result.error, isLoading: false } - : img - ) - ); - } catch (err) { - console.error('Error saving pasted image:', err); - setPastedImages((prev) => - prev.map((img) => - img.id === imageId - ? { ...img, error: 'Failed to save image via Electron.', isLoading: false } - : img - ) - ); - } } }; reader.onerror = () => { @@ -865,25 +810,13 @@ export default function ChatInput({ } }; - // Helper function to handle interruption and queue logic when loading const handleInterruptionAndQueue = () => { if (!isLoading || !hasSubmittableContent) { return false; } - const validPastedImageFilesPaths = pastedImages - .filter((img) => img.filePath && !img.error && !img.isLoading) - .map((img) => img.filePath as string); - const droppedFilePaths = allDroppedFiles - .filter((file) => !file.error && !file.isLoading) - .map((file) => file.path); - - let contentToQueue = displayValue.trim(); - const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths]; - if (allFilePaths.length > 0) { - const pathsString = allFilePaths.join(' '); - contentToQueue = contentToQueue ? `${contentToQueue} ${pathsString}` : pathsString; - } + const imageData = convertImagesToImageData(); + const contentToQueue = appendDroppedFilePaths(displayValue.trim()); const interruptionMatch = detectInterruption(displayValue.trim()); @@ -894,31 +827,25 @@ export default function ChatInput({ // For interruptions, we need to queue the message to be sent after the stop completes // rather than trying to send it immediately while the system is still loading - const interruptionMessage = { + const interruptionMessage: QueuedMessage = { id: Date.now().toString() + Math.random().toString(36).substr(2, 9), content: contentToQueue, timestamp: Date.now(), + images: imageData, }; // Add the interruption message to the front of the queue so it gets sent first setQueuedMessages((prev) => [interruptionMessage, ...prev]); - setDisplayValue(''); - setValue(''); - setPastedImages([]); - if (onFilesProcessed && droppedFiles.length > 0) { - onFilesProcessed(); - } - if (localDroppedFiles.length > 0) { - setLocalDroppedFiles([]); - } + clearInputState(); return true; } - const newMessage = { + const newMessage: QueuedMessage = { id: Date.now().toString() + Math.random().toString(36).substr(2, 9), content: contentToQueue, timestamp: Date.now(), + images: imageData, }; setQueuedMessages((prev) => { const newQueue = [...prev, newMessage]; @@ -929,53 +856,35 @@ export default function ChatInput({ } return newQueue; }); - setDisplayValue(''); - setValue(''); - setPastedImages([]); - if (onFilesProcessed && droppedFiles.length > 0) { - onFilesProcessed(); - } - if (localDroppedFiles.length > 0) { - setLocalDroppedFiles([]); - } + clearInputState(); return true; }; const canSubmit = !isLoading && (displayValue.trim() || - pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) || + pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || allDroppedFiles.some((file) => !file.error && !file.isLoading)); const performSubmit = useCallback( (text?: string) => { - const validPastedImageFilesPaths = pastedImages - .filter((img) => img.filePath && !img.error && !img.isLoading) - .map((img) => img.filePath as string); - // Get paths from all dropped files (both parent and local) - const droppedFilePaths = allDroppedFiles - .filter((file) => !file.error && !file.isLoading) - .map((file) => file.path); - - let textToSend = text ?? displayValue.trim(); - - // Combine pasted images and dropped files - const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths]; - if (allFilePaths.length > 0) { - const pathsString = allFilePaths.join(' '); - textToSend = textToSend ? `${textToSend} ${pathsString}` : pathsString; - } + const imageData = convertImagesToImageData(); + const textToSend = appendDroppedFilePaths(text ?? displayValue.trim()); - if (textToSend) { + if (textToSend || imageData.length > 0) { + // Store original message in history if (displayValue.trim()) { LocalMessageStorage.addMessage(displayValue); - } else if (allFilePaths.length > 0) { - LocalMessageStorage.addMessage(allFilePaths.join(' ')); + } else { + const droppedFilePaths = allDroppedFiles + .filter((file) => !file.isImage && !file.error && !file.isLoading) + .map((file) => file.path); + if (droppedFilePaths.length > 0) { + LocalMessageStorage.addMessage(droppedFilePaths.join(' ')); + } } - handleSubmit( - new CustomEvent('submit', { detail: { value: textToSend } }) as unknown as React.FormEvent - ); + handleSubmit({ msg: textToSend, images: imageData }); // Auto-resume queue after sending a NON-interruption message (if it was paused due to interruption) if ( @@ -988,33 +897,21 @@ export default function ChatInput({ setLastInterruption(null); } - setDisplayValue(''); - setValue(''); - setPastedImages([]); + clearInputState(); setHistoryIndex(-1); setSavedInput(''); setIsInGlobalHistory(false); setHasUserTyped(false); - - // Clear both parent and local dropped files after processing - if (onFilesProcessed && droppedFiles.length > 0) { - onFilesProcessed(); - } - if (localDroppedFiles.length > 0) { - setLocalDroppedFiles([]); - } } }, [ - allDroppedFiles, + convertImagesToImageData, + appendDroppedFilePaths, displayValue, - droppedFiles.length, + allDroppedFiles, handleSubmit, lastInterruption, - localDroppedFiles.length, - onFilesProcessed, - pastedImages, - setLocalDroppedFiles, + clearInputState, ] ); @@ -1090,29 +987,85 @@ export default function ChatInput({ const canSubmit = !isLoading && (displayValue.trim() || - pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) || + pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || allDroppedFiles.some((file) => !file.error && !file.isLoading)); if (canSubmit) { performSubmit(); } }; - const handleFileSelect = async () => { + const fileInputRef = React.useRef(null); + + const handleFileSelect = () => { if (isFilePickerOpen) return; + fileInputRef.current?.click(); + }; + + const handleFileInputChange = async (e: React.ChangeEvent) => { + const files = e.target.files; + if (!files || files.length === 0) return; + setIsFilePickerOpen(true); - try { - const path = await window.electron.selectFileOrDirectory(); - if (path) { - const isDirectory = !path.includes('.') || path.endsWith('/'); - trackFileAttached(isDirectory ? 'directory' : 'file'); + const file = files[0]; + const isImage = file.type.startsWith('image/'); - const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path; - setDisplayValue(newValue); - setValue(newValue); - textAreaRef.current?.focus(); + if (isImage) { + trackFileAttached('file'); + + if (pastedImages.length >= MAX_IMAGES_PER_MESSAGE) { + console.warn(`Maximum ${MAX_IMAGES_PER_MESSAGE} images per message`); + setIsFilePickerOpen(false); + return; } - } finally { - setIsFilePickerOpen(false); + + const uniqueId = `upload-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + setPastedImages((prev) => [ + ...prev, + { + id: uniqueId, + dataUrl: '', + isLoading: true, + error: undefined, + }, + ]); + + const reader = new FileReader(); + reader.onload = async (evt) => { + const dataUrl = evt.target?.result as string; + if (dataUrl) { + const compressedDataUrl = await compressImageDataUrl(dataUrl); + setPastedImages((prev) => + prev.map((img) => + img.id === uniqueId + ? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined } + : img + ) + ); + } + }; + reader.onerror = () => { + setPastedImages((prev) => + prev.map((img) => + img.id === uniqueId + ? { ...img, isLoading: false, error: 'Failed to read image file' } + : img + ) + ); + }; + reader.readAsDataURL(file); + } else { + trackFileAttached('file'); + const path = window.electron.getPathForFile(file); + const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path; + setDisplayValue(newValue); + setValue(newValue); + } + + textAreaRef.current?.focus(); + setIsFilePickerOpen(false); + if (e.target) { + e.target.value = ''; } }; @@ -1140,7 +1093,7 @@ export default function ChatInput({ const hasSubmittableContent = displayValue.trim() || - pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) || + pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || allDroppedFiles.some((file) => !file.error && !file.isLoading); const isAnyImageLoading = pastedImages.some((img) => img.isLoading); const isAnyDroppedFileLoading = allDroppedFiles.some((file) => file.isLoading); @@ -1196,11 +1149,7 @@ export default function ChatInput({ // Remove the message from queue and send it immediately setQueuedMessages((prev) => prev.filter((msg) => msg.id !== messageId)); LocalMessageStorage.addMessage(messageToSend.content); - handleSubmit( - new CustomEvent('submit', { - detail: { value: messageToSend.content }, - }) as unknown as React.FormEvent - ); + handleSubmit({ msg: messageToSend.content, images: messageToSend.images }); // Restore previous pause state after a brief delay to prevent race condition setTimeout(() => { @@ -1214,11 +1163,7 @@ export default function ChatInput({ if (!isLoading && queuedMessages.length > 0) { const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); - handleSubmit( - new CustomEvent('submit', { - detail: { value: nextMessage.content }, - }) as unknown as React.FormEvent - ); + handleSubmit({ msg: nextMessage.content, images: nextMessage.images }); setQueuedMessages((prev) => { const newQueue = prev.slice(1); // If queue becomes empty after processing, clear the paused state @@ -1244,6 +1189,7 @@ export default function ChatInput({ onDrop={handleLocalDrop} onDragOver={handleLocalDragOver} > + {/* Message Queue Display */} {queuedMessages.length > 0 && ( -

+

{img.error.substring(0, 50)}

- {img.dataUrl && ( - - )}
)} {!img.isLoading && ( @@ -1570,7 +1505,7 @@ export default function ChatInput({ - Attach file or directory + Attach file
{/* Model selector, mode selector, alerts, summarize button */} diff --git a/ui/desktop/src/components/ChatSessionsContainer.tsx b/ui/desktop/src/components/ChatSessionsContainer.tsx index 3afac2ce8ded..d7d34566c928 100644 --- a/ui/desktop/src/components/ChatSessionsContainer.tsx +++ b/ui/desktop/src/components/ChatSessionsContainer.tsx @@ -1,10 +1,14 @@ import { useSearchParams } from 'react-router-dom'; import BaseChat from './BaseChat'; import { ChatType } from '../types/chat'; +import { UserInput } from '../types/message'; interface ChatSessionsContainerProps { setChat: (chat: ChatType) => void; - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; + activeSessions: Array<{ + sessionId: string; + initialMessage?: UserInput; + }>; } /** diff --git a/ui/desktop/src/components/GooseMessage.tsx b/ui/desktop/src/components/GooseMessage.tsx index cbcf47fee236..16de3de51cb2 100644 --- a/ui/desktop/src/components/GooseMessage.tsx +++ b/ui/desktop/src/components/GooseMessage.tsx @@ -1,11 +1,10 @@ import { useMemo, useRef } from 'react'; import ImagePreview from './ImagePreview'; -import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils'; import { formatMessageTimestamp } from '../utils/timeUtils'; import MarkdownContent from './MarkdownContent'; import ToolCallWithResponse from './ToolCallWithResponse'; import { - getTextContent, + getTextAndImageContent, getToolRequests, getToolResponses, getToolConfirmationContent, @@ -45,28 +44,25 @@ export default function GooseMessage({ }: GooseMessageProps) { const contentRef = useRef(null); - let textContent = getTextContent(message); + let { textContent, imagePaths } = getTextAndImageContent(message); - const splitChainOfThought = (text: string): { visibleText: string; cotText: string | null } => { + const splitChainOfThought = (text: string): { displayText: string; cotText: string | null } => { const regex = /([\s\S]*?)<\/think>/i; const match = text.match(regex); if (!match) { - return { visibleText: text, cotText: null }; + return { displayText: text, cotText: null }; } const cotRaw = match[1].trim(); - const visibleText = text.replace(regex, '').trim(); + const displayText = text.replace(regex, '').trim(); return { - visibleText, + displayText, cotText: cotRaw || null, }; }; - const { visibleText, cotText } = splitChainOfThought(textContent); - const imagePaths = extractImagePaths(visibleText); - const displayText = - imagePaths.length > 0 ? removeImagePathsFromText(visibleText, imagePaths) : visibleText; + const { displayText, cotText } = splitChainOfThought(textContent); const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]); const toolRequests = getToolRequests(message); @@ -116,11 +112,13 @@ export default function GooseMessage({ )} - {displayText && ( + {(displayText.trim() || imagePaths.length > 0) && (
-
- -
+ {displayText.trim() && ( +
+ +
+ )} {imagePaths.length > 0 && (
diff --git a/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx b/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx index 204c292417ee..13327af6de2c 100644 --- a/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx +++ b/ui/desktop/src/components/GooseSidebar/AppSidebar.tsx @@ -263,7 +263,7 @@ const AppSidebar: React.FC = ({ currentPath }) => { let isPolling = false; const handleSessionCreated = (event: Event) => { - const { session } = (event as CustomEvent<{ session?: Session }>).detail; + const { session } = (event as CustomEvent<{ session?: Session }>).detail || {}; // If session data is provided, add it immediately to the sidebar // This is for displaying sessions that won't be returned by the API due to not having messages yet if (session) { diff --git a/ui/desktop/src/components/Hub.tsx b/ui/desktop/src/components/Hub.tsx index cdd62f6e04d6..a36cff26fc6d 100644 --- a/ui/desktop/src/components/Hub.tsx +++ b/ui/desktop/src/components/Hub.tsx @@ -29,6 +29,7 @@ import { import { getInitialWorkingDir } from '../utils/workingDir'; import { createSession } from '../sessions'; import LoadingGoose from './LoadingGoose'; +import { UserInput } from '../types/message'; export default function Hub({ setView, @@ -39,11 +40,9 @@ export default function Hub({ const [workingDir, setWorkingDir] = useState(getInitialWorkingDir()); const [isCreatingSession, setIsCreatingSession] = useState(false); - const handleSubmit = async (e: React.FormEvent) => { - const customEvent = e as unknown as CustomEvent; - const combinedTextFromInput = customEvent.detail?.value || ''; - - if (combinedTextFromInput.trim() && !isCreatingSession) { + const handleSubmit = async (input: UserInput) => { + const { msg: userMessage, images } = input; + if ((images.length > 0 || userMessage.trim()) && !isCreatingSession) { const extensionConfigs = getExtensionConfigsWithOverrides(extensionsList); clearExtensionOverrides(); setIsCreatingSession(true); @@ -57,21 +56,19 @@ export default function Hub({ window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED)); window.dispatchEvent( new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, { - detail: { sessionId: session.id, initialMessage: combinedTextFromInput }, + detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } }, }) ); setView('pair', { disableAnimation: true, resumeSessionId: session.id, - initialMessage: combinedTextFromInput, + initialMessage: { msg: userMessage, images }, }); } catch (error) { console.error('Failed to create session:', error); setIsCreatingSession(false); } - - e.preventDefault(); } }; diff --git a/ui/desktop/src/components/ImagePreview.tsx b/ui/desktop/src/components/ImagePreview.tsx index 29e0aff33c92..c361ae62ef91 100644 --- a/ui/desktop/src/components/ImagePreview.tsx +++ b/ui/desktop/src/components/ImagePreview.tsx @@ -1,88 +1,32 @@ -import { useState, useEffect } from 'react'; +import { useState } from 'react'; interface ImagePreviewProps { src: string; - alt?: string; - className?: string; } -export default function ImagePreview({ - src, - alt = 'Pasted image', - className = '', -}: ImagePreviewProps) { +export default function ImagePreview({ src }: ImagePreviewProps) { const [isExpanded, setIsExpanded] = useState(false); const [error, setError] = useState(false); - const [isLoading, setIsLoading] = useState(true); - const [imageData, setImageData] = useState(null); - - useEffect(() => { - const loadImage = async () => { - try { - // Use the IPC handler to get the image data - const data = await window.electron.getTempImage(src); - if (data) { - setImageData(data); - setIsLoading(false); - } else { - setError(true); - setIsLoading(false); - } - } catch (err) { - console.error('Error loading image:', err); - setError(true); - setIsLoading(false); - } - }; - - loadImage(); - }, [src]); - - const handleError = () => { - setError(true); - setIsLoading(false); - }; - - const toggleExpand = () => { - if (!error) { - setIsExpanded(!isExpanded); - } - }; - - // Validate that this is a safe file path (should contain goose-pasted-images) - if (!src.includes('goose-pasted-images')) { - return
Invalid image path: {src}
; - } if (error) { - return
Unable to load image: {src}
; + return
Unable to load image
; } return ( -
- {isLoading && ( -
- Loading... -
- )} - {imageData && ( - {alt} - )} - {isExpanded && !error && !isLoading && imageData && ( -
Click to collapse
- )} - {!isExpanded && !error && !isLoading && imageData && ( -
Click to expand
- )} +
+ goose image setError(true)} + onClick={() => setIsExpanded(!isExpanded)} + className={`rounded border border-borderSubtle cursor-pointer hover:border-borderStandard transition-all ${ + isExpanded ? 'max-w-full max-h-96' : 'max-h-40 max-w-40' + }`} + style={{ objectFit: 'contain' }} + /> +
+ Click to {isExpanded ? 'collapse' : 'expand'} +
); } diff --git a/ui/desktop/src/components/Layout/AppLayout.tsx b/ui/desktop/src/components/Layout/AppLayout.tsx index 81bae98e7955..99f721361419 100644 --- a/ui/desktop/src/components/Layout/AppLayout.tsx +++ b/ui/desktop/src/components/Layout/AppLayout.tsx @@ -7,9 +7,13 @@ import { Button } from '../ui/button'; import { Sidebar, SidebarInset, SidebarProvider, SidebarTrigger, useSidebar } from '../ui/sidebar'; import ChatSessionsContainer from '../ChatSessionsContainer'; import { useChatContext } from '../../contexts/ChatContext'; +import { UserInput } from '../../types/message'; interface AppLayoutContentProps { - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; + activeSessions: Array<{ + sessionId: string; + initialMessage?: UserInput; + }>; } const AppLayoutContent: React.FC = ({ activeSessions }) => { @@ -126,7 +130,10 @@ const AppLayoutContent: React.FC = ({ activeSessions }) = }; interface AppLayoutProps { - activeSessions: Array<{ sessionId: string; initialMessage?: string }>; + activeSessions: Array<{ + sessionId: string; + initialMessage?: UserInput; + }>; } export const AppLayout: React.FC = ({ activeSessions }) => { diff --git a/ui/desktop/src/components/MessageQueue.tsx b/ui/desktop/src/components/MessageQueue.tsx index a580939086c9..2eb783ee0987 100644 --- a/ui/desktop/src/components/MessageQueue.tsx +++ b/ui/desktop/src/components/MessageQueue.tsx @@ -1,11 +1,13 @@ import React, { useState } from 'react'; import { X, Clock, Send, GripVertical, Zap, Sparkles, ChevronDown, ChevronUp } from 'lucide-react'; import { Button } from './ui/button'; +import { ImageData } from '../types/message'; -interface QueuedMessage { +export interface QueuedMessage { id: string; content: string; timestamp: number; + images: ImageData[]; } interface MessageQueueProps { diff --git a/ui/desktop/src/components/UserMessage.tsx b/ui/desktop/src/components/UserMessage.tsx index daf71dcf8034..a022fd8f17b6 100644 --- a/ui/desktop/src/components/UserMessage.tsx +++ b/ui/desktop/src/components/UserMessage.tsx @@ -1,8 +1,7 @@ -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import ImagePreview from './ImagePreview'; -import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils'; import MarkdownContent from './MarkdownContent'; -import { getTextContent } from '../types/message'; +import { getTextAndImageContent } from '../types/message'; import { Message } from '../api'; import MessageCopyLink from './MessageCopyLink'; import { formatMessageTimestamp } from '../utils/timeUtils'; @@ -21,35 +20,23 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro const [editContent, setEditContent] = useState(''); const [error, setError] = useState(null); - // Extract text content from the message - const textContent = getTextContent(message); - - // Extract image paths from the message - const imagePaths = extractImagePaths(textContent); - - // Remove image paths from text for display - memoized for performance - const displayText = useMemo( - () => removeImagePathsFromText(textContent, imagePaths), - [textContent, imagePaths] - ); - - // Memoize the timestamp - const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]); + const { textContent, imagePaths } = getTextAndImageContent(message); + const timestamp = formatMessageTimestamp(message.created); // Effect to handle message content changes and ensure persistence useEffect(() => { // If we're not editing, update the edit content to match the current message if (!isEditing) { - setEditContent(displayText); + setEditContent(textContent); } - }, [message.content, displayText, message.id, isEditing]); + }, [message.content, textContent, message.id, isEditing]); // Initialize edit mode with current message content const initializeEditMode = useCallback(() => { - setEditContent(displayText); + setEditContent(textContent); setError(null); - window.electron.logInfo(`Entering edit mode with content: ${displayText}`); - }, [displayText]); + window.electron.logInfo(`Entering edit mode with content: ${textContent}`); + }, [textContent]); // Handle edit button click const handleEditClick = useCallback(() => { @@ -93,7 +80,7 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro setIsEditing(false); - if (editContent.trim() === displayText.trim()) { + if (editContent.trim() === textContent.trim()) { return; } @@ -101,16 +88,16 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro onMessageUpdate(message.id, editContent, editType); } }, - [editContent, displayText, onMessageUpdate, message.id] + [editContent, textContent, onMessageUpdate, message.id] ); // Handle cancel action const handleCancel = useCallback(() => { window.electron.logInfo('Cancel clicked - reverting to original content'); setIsEditing(false); - setEditContent(displayText); // Reset to original content + setEditContent(textContent); // Reset to original content setError(null); - }, [displayText]); + }, [textContent]); // Handle keyboard events for accessibility const handleKeyDown = useCallback( @@ -207,20 +194,21 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
-
-
- + {textContent.trim() && ( +
+
+ +
-
+ )} - {/* Render images if any */} {imagePaths.length > 0 && (
{imagePaths.map((imagePath, index) => ( - + ))}
)} @@ -239,14 +227,14 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro } }} className="flex items-center gap-1 text-xs text-text-subtle hover:cursor-pointer hover:text-text-prominent transition-all duration-200 opacity-0 group-hover:opacity-100 -translate-y-4 group-hover:translate-y-0 focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50 rounded" - aria-label={`Edit message: ${displayText.substring(0, 50)}${displayText.length > 50 ? '...' : ''}`} + aria-label={`Edit message: ${textContent.substring(0, 50)}${textContent.length > 50 ? '...' : ''}`} aria-expanded={isEditing} title="Edit message" > Edit - +
diff --git a/ui/desktop/src/components/sessions/SessionViewComponents.tsx b/ui/desktop/src/components/sessions/SessionViewComponents.tsx index b0188ddc9f39..fb8839c118cc 100644 --- a/ui/desktop/src/components/sessions/SessionViewComponents.tsx +++ b/ui/desktop/src/components/sessions/SessionViewComponents.tsx @@ -7,12 +7,11 @@ import MarkdownContent from '../MarkdownContent'; import ToolCallWithResponse from '../ToolCallWithResponse'; import ImagePreview from '../ImagePreview'; import { - getTextContent, + getTextAndImageContent, ToolRequestMessageContent, ToolResponseMessageContent, } from '../../types/message'; import { formatMessageTimestamp } from '../../utils/timeUtils'; -import { extractImagePaths, removeImagePathsFromText } from '../../utils/imageUtils'; import { Message } from '../../api'; /** @@ -82,15 +81,7 @@ export const SessionMessages: React.FC = ({ ) : messages?.length > 0 ? ( messages .map((message, index) => { - const textContent = getTextContent(message); - // Extract image paths from the message - const imagePaths = extractImagePaths(textContent); - - // Remove image paths from text for display - const displayText = - imagePaths.length > 0 - ? removeImagePathsFromText(textContent, imagePaths) - : textContent; + const { textContent, imagePaths } = getTextAndImageContent(message); // Get tool requests from the message const toolRequests = message.content @@ -128,24 +119,18 @@ export const SessionMessages: React.FC = ({
- {/* Text content */} - {displayText && ( + {textContent && (
0 || imagePaths.length > 0 ? 'mb-4' : ''}`} > - +
)} - {/* Render images if any */} {imagePaths.length > 0 && (
{imagePaths.map((imagePath, imageIndex) => ( - + ))}
)} diff --git a/ui/desktop/src/hooks/useAutoSubmit.ts b/ui/desktop/src/hooks/useAutoSubmit.ts index 05e6df9664c3..5380a9cf48df 100644 --- a/ui/desktop/src/hooks/useAutoSubmit.ts +++ b/ui/desktop/src/hooks/useAutoSubmit.ts @@ -4,6 +4,7 @@ import { useSearchParams } from 'react-router-dom'; import { Session } from '../api'; import { Message } from '../api'; import { ChatState } from '../types/chatState'; +import { UserInput } from '../types/message'; /** * Auto-submit scenarios: @@ -17,8 +18,8 @@ interface UseAutoSubmitProps { session: Session | undefined; messages: Message[]; chatState: ChatState; - initialMessage: string | undefined; - handleSubmit: (message: string) => void; + initialMessage: UserInput | undefined; + handleSubmit: (input: UserInput) => void; } interface UseAutoSubmitReturn { @@ -84,7 +85,7 @@ export function useAutoSubmit({ // Scenario 3: Resume with shouldStartAgent (continue existing conversation) if (shouldStartAgent) { hasAutoSubmittedRef.current = true; - handleSubmit(''); + handleSubmit({ msg: '', images: [] }); } }, [ session, diff --git a/ui/desktop/src/hooks/useChatStream.ts b/ui/desktop/src/hooks/useChatStream.ts index d146be51bf5f..e3ce5e5f0bdb 100644 --- a/ui/desktop/src/hooks/useChatStream.ts +++ b/ui/desktop/src/hooks/useChatStream.ts @@ -21,6 +21,7 @@ import { getCompactingMessage, getThinkingMessage, NotificationEvent, + UserInput, } from '../types/message'; import { errorMessage } from '../utils/conversionUtils'; import { showExtensionLoadResults } from '../utils/extensionErrorUtils'; @@ -39,7 +40,7 @@ interface UseChatStreamReturn { messages: Message[]; chatState: ChatState; setChatState: (state: ChatState) => void; - handleSubmit: (userMessage: string) => Promise; + handleSubmit: (input: UserInput) => Promise; submitElicitationResponse: ( elicitationId: string, userData: Record @@ -449,7 +450,8 @@ export function useChatStream({ }, [sessionId, onSessionLoaded]); const handleSubmit = useCallback( - async (userMessage: string) => { + async (input: UserInput) => { + const { msg: userMessage, images } = input; const currentState = stateRef.current; // Guard: Don't submit if session hasn't been loaded yet @@ -458,7 +460,7 @@ export function useChatStream({ } const hasExistingMessages = currentState.messages.length > 0; - const hasNewMessage = userMessage.trim().length > 0; + const hasNewMessage = userMessage.trim().length > 0 || images.length > 0; // Don't submit if there's no message and no conversation to continue if (!hasNewMessage && !hasExistingMessages) { @@ -520,7 +522,7 @@ export function useChatStream({ } const newMessage = hasNewMessage - ? createUserMessage(userMessage) + ? createUserMessage(userMessage, images) : currentState.messages[currentState.messages.length - 1]; const currentMessages = hasNewMessage ? [...currentState.messages, newMessage] @@ -697,7 +699,7 @@ export function useChatStream({ if (sessionResponse.data?.conversation) { dispatch({ type: 'SET_MESSAGES', payload: sessionResponse.data.conversation }); } - await handleSubmit(newContent); + await handleSubmit({ msg: newContent, images: [] }); } } catch (error) { const errorMsg = errorMessage(error); diff --git a/ui/desktop/src/hooks/useFileDrop.ts b/ui/desktop/src/hooks/useFileDrop.ts index d88e2e77381b..40de5e012b1b 100644 --- a/ui/desktop/src/hooks/useFileDrop.ts +++ b/ui/desktop/src/hooks/useFileDrop.ts @@ -1,4 +1,5 @@ import { useCallback, useState, useRef, useEffect } from 'react'; +import { compressImageDataUrl } from '../utils/conversionUtils'; export interface DroppedFile { id: string; @@ -6,7 +7,7 @@ export interface DroppedFile { name: string; type: string; isImage: boolean; - dataUrl?: string; // For image previews + dataUrl?: string; isLoading?: boolean; error?: string; } @@ -71,25 +72,41 @@ export const useFileDrop = () => { droppedFileObjects.push(droppedFile); - // For images, generate a preview (only if successfully processed) if (droppedFile.isImage && !droppedFile.error) { const reader = new FileReader(); activeReadersRef.current.add(reader); - reader.onload = (event) => { + reader.onload = async (event) => { const dataUrl = event.target?.result as string; - setDroppedFiles((prev) => - prev.map((f) => (f.id === droppedFile.id ? { ...f, dataUrl, isLoading: false } : f)) - ); + try { + // Compress the image + const compressedDataUrl = await compressImageDataUrl(dataUrl); + setDroppedFiles((prev) => + prev.map((f) => + f.id === droppedFile.id + ? { ...f, dataUrl: compressedDataUrl, isLoading: false } + : f + ) + ); + } catch (compressionError) { + console.error('Failed to compress image:', file.name, compressionError); + setDroppedFiles((prev) => + prev.map((f) => + f.id === droppedFile.id + ? { ...f, error: 'Failed to compress image', isLoading: false } + : f + ) + ); + } activeReadersRef.current.delete(reader); }; reader.onerror = () => { - console.error('Failed to generate preview for:', file.name); + console.error('Failed to read image:', file.name); setDroppedFiles((prev) => prev.map((f) => f.id === droppedFile.id - ? { ...f, error: 'Failed to load image preview', isLoading: false } + ? { ...f, error: 'Failed to load image', isLoading: false } : f ) ); diff --git a/ui/desktop/src/main.ts b/ui/desktop/src/main.ts index 1d7e399dfdb8..10293068c056 100644 --- a/ui/desktop/src/main.ts +++ b/ui/desktop/src/main.ts @@ -1576,79 +1576,6 @@ ipcMain.handle('save-data-url-to-temp', async (_event, dataUrl: string, uniqueId } }); -// IPC handler to serve temporary image files -ipcMain.handle('get-temp-image', async (_event, filePath: string) => { - console.log(`[Main] Received get-temp-image for path: ${filePath}`); - - // Input validation - if (!filePath || typeof filePath !== 'string') { - console.warn('[Main] Invalid file path provided for image serving'); - return null; - } - - // Ensure the path is within the designated temp directory - const resolvedPath = path.resolve(filePath); - const resolvedTempDir = path.resolve(gooseTempDir); - - if (!resolvedPath.startsWith(resolvedTempDir + path.sep)) { - console.warn(`[Main] Attempted to access file outside designated temp directory: ${filePath}`); - return null; - } - - try { - // Check if it's a regular file first, before trying realpath - const stats = await fs.lstat(filePath); - if (!stats.isFile()) { - console.warn(`[Main] Not a regular file, refusing to serve: ${filePath}`); - return null; - } - - // Get the real paths for both the temp directory and the file to handle symlinks properly - let realTempDir: string; - let actualPath = filePath; - - try { - realTempDir = await fs.realpath(gooseTempDir); - const realPath = await fs.realpath(filePath); - - // Double-check that the real path is still within our real temp directory - if (!realPath.startsWith(realTempDir + path.sep)) { - console.warn( - `[Main] Real path is outside designated temp directory: ${realPath} not in ${realTempDir}` - ); - return null; - } - actualPath = realPath; - } catch (realpathError) { - // If realpath fails, use the original path validation - console.log( - `[Main] realpath failed for ${filePath}, using original path validation:`, - realpathError instanceof Error ? realpathError.message : String(realpathError) - ); - } - - // Read the file and return as base64 data URL - const fileBuffer = await fs.readFile(actualPath); - const fileExtension = path.extname(actualPath).toLowerCase().substring(1); - - // Validate file extension - const allowedExtensions = ['png', 'jpg', 'jpeg', 'gif', 'webp']; - if (!allowedExtensions.includes(fileExtension)) { - console.warn(`[Main] Unsupported file extension: ${fileExtension}`); - return null; - } - - const mimeType = fileExtension === 'jpg' ? 'image/jpeg' : `image/${fileExtension}`; - const base64Data = fileBuffer.toString('base64'); - const dataUrl = `data:${mimeType};base64,${base64Data}`; - - console.log(`[Main] Served temp image: ${filePath}`); - return dataUrl; - } catch (error) { - console.error(`[Main] Failed to serve temp image: ${filePath}`, error); - return null; - } -}); ipcMain.on('delete-temp-file', async (_event, filePath: string) => { console.log(`[Main] Received delete-temp-file for path: ${filePath}`); diff --git a/ui/desktop/src/preload.ts b/ui/desktop/src/preload.ts index 2f93efcad6f6..bf3b6a4378b6 100644 --- a/ui/desktop/src/preload.ts +++ b/ui/desktop/src/preload.ts @@ -121,8 +121,6 @@ type ElectronAPI = { deleteTempFile: (filePath: string) => void; // Function for opening external URLs securely openExternal: (url: string) => Promise; - // Function to serve temp images - getTempImage: (filePath: string) => Promise; // Update-related functions getVersion: () => string; checkForUpdates: () => Promise<{ updateInfo: unknown; error: string | null }>; @@ -246,9 +244,6 @@ const electronAPI: ElectronAPI = { openExternal: (url: string): Promise => { return ipcRenderer.invoke('open-external', url); }, - getTempImage: (filePath: string): Promise => { - return ipcRenderer.invoke('get-temp-image', filePath); - }, getVersion: (): string => { return config.GOOSE_VERSION || ipcRenderer.sendSync('get-app-version') || ''; }, diff --git a/ui/desktop/src/sessions.ts b/ui/desktop/src/sessions.ts index 1ef635ed712e..d3e87967ba68 100644 --- a/ui/desktop/src/sessions.ts +++ b/ui/desktop/src/sessions.ts @@ -88,9 +88,11 @@ export async function startNewSession( // Include session data so sidebar can add it immediately (before it has messages) window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED, { detail: { session } })); + const initialMessage = initialText ? { msg: initialText, images: [] } : undefined; + const eventDetail = { sessionId: session.id, - initialMessage: initialText, + initialMessage, }; window.dispatchEvent( @@ -101,7 +103,7 @@ export async function startNewSession( setView('pair', { disableAnimation: true, - initialMessage: initialText, + initialMessage, resumeSessionId: session.id, }); return session; diff --git a/ui/desktop/src/types/message.ts b/ui/desktop/src/types/message.ts index ba4425fddeaf..43423578ede3 100644 --- a/ui/desktop/src/types/message.ts +++ b/ui/desktop/src/types/message.ts @@ -7,12 +7,38 @@ export type NotificationEvent = Extract; // Compaction response message - must match backend constant const COMPACTION_THINKING_TEXT = 'goose is compacting the conversation...'; -export function createUserMessage(text: string): Message { +export interface ImageData { + data: string; // base64 encoded image data + mimeType: string; +} + +export interface UserInput { + msg: string; + images: ImageData[]; +} + +export function createUserMessage(text: string, images?: ImageData[]): Message { + const content: Message['content'] = []; + + if (text.trim()) { + content.push({ type: 'text', text }); + } + + if (images && images.length > 0) { + images.forEach((img) => { + content.push({ + type: 'image', + data: img.data, + mimeType: img.mimeType, + }); + }); + } + return { id: generateMessageId(), role: 'user', created: Math.floor(Date.now() / 1000), - content: [{ type: 'text', text }], + content, metadata: { userVisible: true, agentVisible: true }, }; } @@ -43,13 +69,22 @@ export function generateMessageId(): string { return Math.random().toString(36).substring(2, 10); } -export function getTextContent(message: Message): string { - return message.content - .map((content) => { - if (content.type === 'text') return content.text; - return ''; - }) - .join(''); +export function getTextAndImageContent(message: Message): { + textContent: string; + imagePaths: string[]; +} { + let textContent = ''; + const imagePaths: string[] = []; + + for (const content of message.content) { + if (content.type === 'text') { + textContent += content.text; + } else if (content.type === 'image') { + imagePaths.push(`data:${content.mimeType};base64,${content.data}`); + } + } + + return { textContent, imagePaths }; } export function getToolRequests(message: Message): (ToolRequest & { type: 'toolRequest' })[] { diff --git a/ui/desktop/src/utils/conversionUtils.ts b/ui/desktop/src/utils/conversionUtils.ts index 792df32a4e88..4e6a2fce8a5f 100644 --- a/ui/desktop/src/utils/conversionUtils.ts +++ b/ui/desktop/src/utils/conversionUtils.ts @@ -22,6 +22,32 @@ export function errorMessage(err: Error | unknown, default_value?: string) { } } +export async function compressImageDataUrl(dataUrl: string): Promise { + return new Promise((resolve, reject) => { + const img = new globalThis.Image(); + img.onload = () => { + const maxDim = 1024; + const scale = Math.min(1, maxDim / Math.max(img.width, img.height)); + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); + + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (!ctx) { + reject(new Error('Failed to get canvas context')); + return; + } + ctx.drawImage(img, 0, 0, width, height); + + resolve(canvas.toDataURL('image/jpeg', 0.85)); + }; + img.onerror = () => reject(new Error('Failed to load image')); + img.src = dataUrl; + }); +} + export function formatAppName(name: string): string { return name .split(/[-_\s]+/) diff --git a/ui/desktop/src/utils/imageUtils.ts b/ui/desktop/src/utils/imageUtils.ts deleted file mode 100644 index 83bb7338c1b9..000000000000 --- a/ui/desktop/src/utils/imageUtils.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Utility functions for detecting and handling image paths in messages - */ - -/** - * Extracts image file paths from a message text - * Looks for paths that match the pattern of pasted images from the temp directory - * - * @param text The message text to extract image paths from - * @returns An array of image file paths found in the message - */ -export function extractImagePaths(text: string): string[] { - if (!text) return []; - - // Match paths that look like pasted image paths from the temp directory - // Pattern: /path/to/goose-pasted-images/pasted-img-TIMESTAMP-RANDOM.ext - // This regex looks for: - // - Word boundary or start of string - // - A path containing "goose-pasted-images" - // - Followed by a filename starting with "pasted-" - // - Ending with common image extensions - // - Word boundary or end of string - const regex = - /(?:^|\s)((?:[^\s]*\/)?goose-pasted-images\/pasted-[^\s]+\.(png|jpg|jpeg|gif|webp))(?=\s|$)/gi; - - const matches = []; - let match; - - while ((match = regex.exec(text)) !== null) { - matches.push(match[1]); - } - - return matches; -} - -/** - * Removes image paths from the text - * - * @param text The original text - * @param imagePaths Array of image paths to remove - * @returns Text with image paths removed - */ -export function removeImagePathsFromText(text: string, imagePaths: string[]): string { - if (!text || imagePaths.length === 0) return text; - - let result = text; - - // Remove each image path from the text - imagePaths.forEach((path) => { - // Escape special regex characters in the path - const escapedPath = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - // Create a regex that matches the path with optional surrounding whitespace - const pathRegex = new RegExp(`(^|\\s)${escapedPath}(?=\\s|$)`, 'g'); - result = result.replace(pathRegex, '$1'); - }); - - // Clean up any extra whitespace - return result.replace(/\s+/g, ' ').trim(); -} diff --git a/ui/desktop/src/utils/navigationUtils.ts b/ui/desktop/src/utils/navigationUtils.ts index bb846291bf42..33f29f068ef6 100644 --- a/ui/desktop/src/utils/navigationUtils.ts +++ b/ui/desktop/src/utils/navigationUtils.ts @@ -1,5 +1,6 @@ import { NavigateFunction } from 'react-router-dom'; -import { Recipe } from '../api/types.gen'; +import { Recipe } from '../api'; +import { UserInput } from '../types/message'; export type View = | 'welcome' @@ -29,7 +30,7 @@ export type ViewOptions = { parentView?: View; parentViewOptions?: ViewOptions; disableAnimation?: boolean; - initialMessage?: string; + initialMessage?: UserInput; shareToken?: string; resumeSessionId?: string; pendingScheduleDeepLink?: string; diff --git a/ui/desktop/src/utils/toolCallChaining.ts b/ui/desktop/src/utils/toolCallChaining.ts index d676da874433..3b6a5a785392 100644 --- a/ui/desktop/src/utils/toolCallChaining.ts +++ b/ui/desktop/src/utils/toolCallChaining.ts @@ -1,4 +1,4 @@ -import { getToolRequests, getTextContent, getToolResponses } from '../types/message'; +import { getToolRequests, getTextAndImageContent, getToolResponses } from '../types/message'; import { Message } from '../api'; export function identifyConsecutiveToolCalls(messages: Message[]): number[][] { @@ -9,7 +9,7 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] { const message = messages[i]; const toolRequests = getToolRequests(message); const toolResponses = getToolResponses(message); - const textContent = getTextContent(message); + const { textContent } = getTextAndImageContent(message); const hasText = textContent.trim().length > 0; if (toolResponses.length > 0 && toolRequests.length === 0) { @@ -47,15 +47,6 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] { return chains; } -export function shouldHideMessage(messageIndex: number, chains: number[][]): boolean { - for (const chain of chains) { - if (chain.includes(messageIndex)) { - return chain[0] !== messageIndex; - } - } - return false; -} - export function shouldHideTimestamp(messageIndex: number, chains: number[][]): boolean { for (const chain of chains) { if (chain.includes(messageIndex)) { @@ -69,7 +60,3 @@ export function shouldHideTimestamp(messageIndex: number, chains: number[][]): b export function isInChain(messageIndex: number, chains: number[][]): boolean { return chains.some((chain) => chain.includes(messageIndex)); } - -export function getChainForMessage(messageIndex: number, chains: number[][]): number[] | null { - return chains.find((chain) => chain.includes(messageIndex)) || null; -}