Native images

ui/desktop/src/App.tsx

@@ -20,10 +20,11 @@ import { createSession } from './sessions';
 
 import { ChatType } from './types/chat';
 import Hub from './components/Hub';
+import { UserInput } from './types/message';
 
 interface PairRouteState {
   resumeSessionId?: string;
-  initialMessage?: string;
+  initialMessage?: UserInput;
 }
 import SettingsView, { SettingsViewOptions } from './components/settings/SettingsView';
 import SessionsView from './components/sessions/SessionsView';
@@ -68,8 +69,13 @@ const HubRouteWrapper = () => {
 const PairRouteWrapper = ({
   activeSessions,
 }: {
-  activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
-  setActiveSessions: (sessions: Array<{ sessionId: string; initialMessage?: string }>) => void;
+  activeSessions: Array<{
+    sessionId: string;
+    initialMessage?: UserInput;
+  }>;
+  setActiveSessions: (
+    sessions: Array<{ sessionId: string; initialMessage?: UserInput }>
+  ) => void;
 }) => {
   const { extensionsList } = useConfig();
   const location = useLocation();
@@ -354,13 +360,16 @@ export function AppInner() {
   const MAX_ACTIVE_SESSIONS = 10;
 
   const [activeSessions, setActiveSessions] = useState<
-    Array<{ sessionId: string; initialMessage?: string }>
+    Array<{ sessionId: string; initialMessage?: UserInput }>
   >([]);
 
   useEffect(() => {
     const handleAddActiveSession = (event: Event) => {
       const { sessionId, initialMessage } = (
-        event as CustomEvent<{ sessionId: string; initialMessage?: string }>
+        event as CustomEvent<{
+          sessionId: string;
+          initialMessage?: UserInput;
+        }>
       ).detail;
 
       setActiveSessions((prev) => {

ui/desktop/src/components/BaseChat.tsx

@@ -28,17 +28,18 @@ import { useNavigation } from '../hooks/useNavigation';
 import { RecipeHeader } from './RecipeHeader';
 import { RecipeWarningModal } from './ui/RecipeWarningModal';
 import { scanRecipe } from '../recipe';
+import { UserInput } from '../types/message';
 import { useCostTracking } from '../hooks/useCostTracking';
 import RecipeActivities from './recipes/RecipeActivities';
 import { useToolCount } from './alerts/useToolCount';
-import { getThinkingMessage, getTextContent } from '../types/message';
+import { getThinkingMessage, getTextAndImageContent } from '../types/message';
 import ParameterInputModal from './ParameterInputModal';
 import { substituteParameters } from '../utils/providerUtils';
 import CreateRecipeFromSessionModal from './recipes/CreateRecipeFromSessionModal';
 import { toastSuccess } from '../toasts';
 import { Recipe } from '../recipe';
 import { useAutoSubmit } from '../hooks/useAutoSubmit';
-import { Goose } from './icons/Goose';
+import { Goose } from './icons';
 import EnvironmentBadge from './GooseSidebar/EnvironmentBadge';
 
 const CurrentModelContext = createContext<{ model: string; mode: string } | null>(null);
@@ -56,10 +57,10 @@ interface BaseChatProps {
   suppressEmptyState: boolean;
   sessionId: string;
   isActiveSession: boolean;
-  initialMessage?: string;
+  initialMessage?: UserInput;
 }
 
-function BaseChatContent({
+export default function BaseChat({
   setChat,
   renderHeader,
   customChatInputProps = {},
@@ -149,7 +150,7 @@ function BaseChatContent({
     return messages
       .reduce<string[]>((history, message) => {
         if (message.role === 'user') {
-          const text = getTextContent(message).trim();
+          const text = getTextAndImageContent(message).textContent.trim();
           if (text) {
             history.push(text);
           }
@@ -159,14 +160,11 @@ function BaseChatContent({
       .reverse();
   }, [messages]);
 
-  const handleFormSubmit = (e: React.FormEvent) => {
-    const customEvent = e as unknown as CustomEvent;
-    const textValue = customEvent.detail?.value || '';
-
-    if (recipe && textValue.trim()) {
+  const chatInputSubmit = (input: UserInput) => {
+    if (recipe && input.msg.trim()) {
       setHasStartedUsingRecipe(true);
     }
-    handleSubmit(textValue);
+    handleSubmit(input);
   };
 
   const { sessionCosts } = useCostTracking({
@@ -326,7 +324,6 @@ function BaseChatContent({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [session?.name, setChat]);
 
-  // Only use initialMessage for the prompt if it hasn't been submitted yet
   // If we have a recipe prompt and user recipe values, substitute parameters
   let recipePrompt = '';
   if (messages.length === 0 && recipe?.prompt) {
@@ -413,7 +410,7 @@ function BaseChatContent({
             {recipe && (
               <div className={hasStartedUsingRecipe ? 'mb-6' : ''}>
                 <RecipeActivities
-                  append={(text: string) => handleSubmit(text)}
+                  append={(text: string) => handleSubmit({ msg: text, images: [] })}
                   activities={Array.isArray(recipe.activities) ? recipe.activities : null}
                   title={recipe.title}
                   parameterValues={session?.user_recipe_values || {}}
@@ -428,7 +425,7 @@ function BaseChatContent({
                     messages={messages}
                     chat={{ sessionId }}
                     toolCallNotifications={toolCallNotifications}
-                    append={(text: string) => handleSubmit(text)}
+                    append={(text: string) => handleSubmit({ msg: text, images: [] })}
                     isUserMessage={(m: Message) => m.role === 'user'}
                     isStreamingMessage={chatState !== ChatState.Idle}
                     onRenderingComplete={handleRenderingComplete}
@@ -440,7 +437,7 @@ function BaseChatContent({
                 <div className="block h-8" />
               </>
             ) : !recipe && showPopularTopics ? (
-              <PopularChatTopics append={(text: string) => handleSubmit(text)} />
+              <PopularChatTopics append={(text: string) => handleSubmit({ msg: text, images: [] })} />
             ) : null}
           </ScrollArea>
 
@@ -464,7 +461,7 @@ function BaseChatContent({
           <ChatInput
             inputRef={chatInputRef}
             sessionId={sessionId}
-            handleSubmit={handleFormSubmit}
+            handleSubmit={chatInputSubmit}
             chatState={chatState}
             setChatState={setChatState}
             onStop={stopStreaming}
@@ -527,7 +524,3 @@ function BaseChatContent({
     </div>
   );
 }
-
-export default function BaseChat(props: BaseChatProps) {
-  return <BaseChatContent {...props} />;
-}