diff --git a/web/packages/common/src/components/AssistantChat/AssistantChatMessageContent.tsx b/web/packages/common/src/components/AssistantChat/AssistantChatMessageContent.tsx new file mode 100644 index 0000000000..57557e138f --- /dev/null +++ b/web/packages/common/src/components/AssistantChat/AssistantChatMessageContent.tsx @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { MessagePrimitive } from '@assistant-ui/react'; +import type { MessageRenderProps } from '@nemo/common/src/components/AssistantChat/types'; +import { MessageContent } from '@nemo/common/src/components/Chat/MessageContent'; +import { Banner } from '@nvidia/foundations-react-core'; + +export const AssistantChatMessageContent = ({ + messageContentProps, + toolCallPartComponent, +}: MessageRenderProps) => ( + <> + , + Image: ({ image, filename }) => ( + {filename + ), + tools: { Fallback: toolCallPartComponent }, + }} + /> + + + There was an error generating a response. + + + +); diff --git a/web/packages/common/src/components/AssistantChat/AssistantChatThread.tsx b/web/packages/common/src/components/AssistantChat/AssistantChatThread.tsx index 54e49741fe..b9cab3c365 100644 --- a/web/packages/common/src/components/AssistantChat/AssistantChatThread.tsx +++ b/web/packages/common/src/components/AssistantChat/AssistantChatThread.tsx @@ -1,84 +1,21 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { ThreadPrimitive } from '@assistant-ui/react'; +import { AssistantComposer } from '@nemo/common/src/components/AssistantChat/AssistantComposer'; +import { AssistantMessage } from '@nemo/common/src/components/AssistantChat/AssistantMessage'; import { - ActionBarPrimitive, - ComposerPrimitive, - MessagePrimitive, - ThreadPrimitive, - type ToolCallMessagePartComponent, -} from '@assistant-ui/react'; -import { ComposerMode } from '@nemo/common/src/components/AssistantChat/types'; + ComposerMode, + type AssistantChatThreadProps, +} from '@nemo/common/src/components/AssistantChat/types'; +import { UserEditComposer } from '@nemo/common/src/components/AssistantChat/UserEditComposer'; +import { UserMessage } from '@nemo/common/src/components/AssistantChat/UserMessage'; import { ChatEmptyState } from '@nemo/common/src/components/Chat/ChatEmptyState'; -import { - MessageContent, - type MessageContentProps, -} from '@nemo/common/src/components/Chat/MessageContent'; -import { - Banner, - Button, - Flex, - Skeleton, - Stack, - Text, - TextArea, - Tooltip, -} from '@nvidia/foundations-react-core'; +import { Flex, Stack } from '@nvidia/foundations-react-core'; import cn from 'classnames'; -import { ArrowUp, Check, Copy, Pencil, RefreshCw, RotateCcw, Square, X } from 'lucide-react'; -import { useCallback, useMemo, type ComponentProps, type ReactNode } from 'react'; - -export interface AssistantChatThreadAttributes { - ThreadViewport?: ComponentProps; -} +import { useCallback, useMemo } from 'react'; -type AssistantChatMessageContentProps = Pick; - -interface AssistantChatThreadProps { - disabled?: boolean; - placeholder: string; - onReset: () => void; - showRunningIndicator?: boolean; - attributes?: AssistantChatThreadAttributes; - composerMode?: ComposerMode; - slotComposerStart?: ReactNode; - emptyState?: { - slotHeading?: string; - slotSubheading?: string; - }; - contentClassName?: string; - composerContainerClassName?: string; - hideAssistantMessageActions?: boolean; - toolCallPartComponent?: ToolCallMessagePartComponent; - viewportClassName?: string; - composerOverride?: ReactNode; - messageContentProps?: AssistantChatMessageContentProps; -} - -const AssistantChatMessageContent = ({ - messageContentProps, - toolCallPartComponent, -}: { - messageContentProps?: AssistantChatMessageContentProps; - toolCallPartComponent?: ToolCallMessagePartComponent; -}) => ( - <> - , - tools: { Fallback: toolCallPartComponent }, - }} - /> - - - There was an error generating a response. - - - -); - -const ACTION_BUTTON_CLASS = - 'flex cursor-pointer size-8 items-center justify-center rounded text-base bg-surface-raised hover:bg-surface-sunken disabled:cursor-not-allowed disabled:opacity-50'; +export type { AssistantChatThreadAttributes } from '@nemo/common/src/components/AssistantChat/types'; const CHAT_VIEWPORT_SCROLLBAR_CLASS = [ '[scrollbar-width:thin]', @@ -91,221 +28,6 @@ const CHAT_VIEWPORT_SCROLLBAR_CLASS = [ '[&::-webkit-scrollbar-thumb:hover]:bg-[var(--border-color-interaction-strong)]', ].join(' '); -const CopyAction = () => ( - - - - - - - - - - -); - -const AssistantMessage = ({ - hideAssistantMessageActions, - messageContentProps, - showRunningIndicator = true, - toolCallPartComponent, -}: { - hideAssistantMessageActions?: boolean; - messageContentProps?: AssistantChatMessageContentProps; - showRunningIndicator?: boolean; - toolCallPartComponent?: ToolCallMessagePartComponent; -}) => ( - - - {showRunningIndicator ? ( - - -
- -
-
-
- ) : null} - {!hideAssistantMessageActions ? ( -
- - - - - - - - -
- ) : null} -
-); - -const UserMessage = ({ - messageContentProps, - toolCallPartComponent, -}: { - messageContentProps?: AssistantChatMessageContentProps; - toolCallPartComponent?: ToolCallMessagePartComponent; -}) => ( - -
- -
-
- - - - - - - - -
-
-); - -const UserEditComposer = () => ( - - - - - - - - - - - - - } - /> - } - /> - - -); - -type AssistantComposerProps = Pick< - AssistantChatThreadProps, - 'disabled' | 'placeholder' | 'onReset' | 'slotComposerStart' -> & { - className?: string; -}; - -const AssistantComposer = ({ - disabled, - placeholder, - onReset, - slotComposerStart, - className, -}: AssistantComposerProps) => ( -
- {slotComposerStart &&
{slotComposerStart}
} - - - - - - - - - - - - - - - - -
-); - export const AssistantChatThread = ({ disabled, placeholder, @@ -322,6 +44,7 @@ export const AssistantChatThread = ({ viewportClassName, composerOverride, messageContentProps, + enableImageAttachments, }: AssistantChatThreadProps) => { const { className: threadViewportClassName, ...threadViewportAttributes } = attributes?.ThreadViewport ?? {}; @@ -345,14 +68,18 @@ export const AssistantChatThread = ({ ), [messageContentProps, toolCallPartComponent] ); + const UserEditComposerWithAttachments = useCallback( + () => , + [enableImageAttachments] + ); const messageComponents = useMemo( () => ({ AssistantMessage: AssistantMessageWithToolCallPart, UserMessage: UserMessageWithToolCallPart, - UserEditComposer, + UserEditComposer: UserEditComposerWithAttachments, SystemMessage: AssistantMessageWithToolCallPart, }), - [AssistantMessageWithToolCallPart, UserMessageWithToolCallPart] + [AssistantMessageWithToolCallPart, UserMessageWithToolCallPart, UserEditComposerWithAttachments] ); return ( @@ -394,6 +121,7 @@ export const AssistantChatThread = ({ placeholder={placeholder} onReset={onReset} slotComposerStart={slotComposerStart} + enableImageAttachments={enableImageAttachments} /> )} diff --git a/web/packages/common/src/components/AssistantChat/AssistantComposer.tsx b/web/packages/common/src/components/AssistantChat/AssistantComposer.tsx new file mode 100644 index 0000000000..616283080c --- /dev/null +++ b/web/packages/common/src/components/AssistantChat/AssistantComposer.tsx @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ComposerPrimitive, ThreadPrimitive } from '@assistant-ui/react'; +import { ComposerAttachmentsRow } from '@nemo/common/src/components/AssistantChat/ComposerAttachments'; +import type { AssistantChatThreadProps } from '@nemo/common/src/components/AssistantChat/types'; +import { Button, Tooltip } from '@nvidia/foundations-react-core'; +import cn from 'classnames'; +import { ArrowUp, ImagePlus, RotateCcw, Square } from 'lucide-react'; + +type AssistantComposerProps = Pick< + AssistantChatThreadProps, + 'disabled' | 'placeholder' | 'onReset' | 'slotComposerStart' | 'enableImageAttachments' +> & { + className?: string; +}; + +export const AssistantComposer = ({ + disabled, + placeholder, + onReset, + slotComposerStart, + className, + enableImageAttachments = true, +}: AssistantComposerProps) => ( +
+ {slotComposerStart &&
{slotComposerStart}
} + + {enableImageAttachments && } +
+ + {enableImageAttachments && ( + + + + + + )} + + + + + + + + + + + + + +
+
+
+); diff --git a/web/packages/common/src/components/AssistantChat/AssistantMessage.tsx b/web/packages/common/src/components/AssistantChat/AssistantMessage.tsx new file mode 100644 index 0000000000..c28f94dab8 --- /dev/null +++ b/web/packages/common/src/components/AssistantChat/AssistantMessage.tsx @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ActionBarPrimitive, MessagePrimitive, ThreadPrimitive } from '@assistant-ui/react'; +import { AssistantChatMessageContent } from '@nemo/common/src/components/AssistantChat/AssistantChatMessageContent'; +import { + ACTION_BUTTON_CLASS, + CopyAction, + MESSAGE_ACTIONS_CLASS, +} from '@nemo/common/src/components/AssistantChat/messageActions'; +import type { MessageRenderProps } from '@nemo/common/src/components/AssistantChat/types'; +import { Skeleton, Tooltip } from '@nvidia/foundations-react-core'; +import { RefreshCw } from 'lucide-react'; + +export const AssistantMessage = ({ + hideAssistantMessageActions, + messageContentProps, + showRunningIndicator = true, + toolCallPartComponent, +}: MessageRenderProps & { + hideAssistantMessageActions?: boolean; + showRunningIndicator?: boolean; +}) => ( + + + {showRunningIndicator ? ( + + +
+ +
+
+
+ ) : null} + {!hideAssistantMessageActions ? ( +
+ + + + + + + + +
+ ) : null} +
+); diff --git a/web/packages/common/src/components/AssistantChat/ComposerAttachments.tsx b/web/packages/common/src/components/AssistantChat/ComposerAttachments.tsx new file mode 100644 index 0000000000..df97ec4c52 --- /dev/null +++ b/web/packages/common/src/components/AssistantChat/ComposerAttachments.tsx @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type Attachment, AttachmentPrimitive, ComposerPrimitive } from '@assistant-ui/react'; +import { Flex, Tooltip } from '@nvidia/foundations-react-core'; +import { Image as ImageIcon, X } from 'lucide-react'; +import { useEffect, useState } from 'react'; + +const useImagePreviewUrl = (file?: File): string | undefined => { + const [url, setUrl] = useState(); + useEffect(() => { + if (!file) { + setUrl(undefined); + return undefined; + } + const objectUrl = URL.createObjectURL(file); + setUrl(objectUrl); + return () => { + URL.revokeObjectURL(objectUrl); + setUrl(undefined); + }; + }, [file]); + return url; +}; + +const ComposerAttachmentChip = ({ attachment }: { attachment: Attachment }) => { + const liftedImage = attachment.content?.find((part) => part.type === 'image')?.image; + const previewFromFile = useImagePreviewUrl(attachment.file); + const previewUrl = previewFromFile ?? liftedImage; + return ( + + + {previewUrl ? ( + {attachment.name} + ) : ( +
+ +
+ )} + + + +
+
+ ); +}; + +/** + * Renders the current composer's attachments as a wrapping row of thumbnail + * chips. Collapses to nothing (`empty:hidden`) when there are no attachments. + * Reads from the surrounding `ComposerPrimitive.Root`, so it works for both the + * main prompt composer and the inline edit composer. + */ +export const ComposerAttachmentsRow = () => ( + + + {({ attachment }) => } + + +); diff --git a/web/packages/common/src/components/AssistantChat/README.md b/web/packages/common/src/components/AssistantChat/README.md index 20ad65eb39..1252e09e13 100644 --- a/web/packages/common/src/components/AssistantChat/README.md +++ b/web/packages/common/src/components/AssistantChat/README.md @@ -66,7 +66,8 @@ Related documentation: ## Supported -- Text-only user prompts with Enter-to-send and Shift+Enter newlines. +- Text user prompts with Enter-to-send and Shift+Enter newlines. +- Image attachments (pick or paste) forwarded as OpenAI multimodal content, enabled only when `enableImageAttachments` is true and the selected model supports image attachments. - Streaming assistant responses through `useChatCompletion` with `model`, `workspace`, or explicit `baseURL`. - Optional `PromptData` support for system prompt, temperature, and max tokens. - Optional OpenAI-compatible tool definitions on the completion request. @@ -75,9 +76,21 @@ Related documentation: - Basic assistant-ui runtime hooks for future copy, branch, and thread-list features. - Storybook MSW mocks for normal streaming, indexed example responses, and a hanging stream used to validate Stop behavior. +## File Layout + +- `index.tsx` — `AssistantChat` entry point; instantiates the runtime and renders the thread. +- `useAssistantChatRuntime.ts` — assistant-ui `ExternalStoreRuntime`, message conversion, and edit/reload/cancel/reset handlers. +- `AssistantChatThread.tsx` — thread viewport layout that wires message renderers and the composer together. +- `AssistantMessage.tsx` / `UserMessage.tsx` — assistant/system and user message renderers (`AssistantMessage` also renders system messages). +- `UserEditComposer.tsx` — the inline composer shown when editing a user message. +- `AssistantChatMessageContent.tsx` — shared message-parts renderer (text, image, tool-call fallback, error banner) used by both message renderers. +- `messageActions.tsx` — shared message-action styles and the copy-message button. +- `AssistantComposer.tsx` — the prompt composer, send/stop/reset controls, and image-attachment chips. +- `types.ts` — shared prop and config types. `messageUtils.ts` / `completionUtils.ts` — message and completion helpers. + ## Not Yet Supported -- File attachments and multimodal message parts. +- Non-image file attachments (documents, audio). - Persisted multi-thread history or server-backed thread management. - Feedback/intake submission parity with the existing `Chat` component. - Custom tool call result UIs. diff --git a/web/packages/common/src/components/AssistantChat/UserEditComposer.tsx b/web/packages/common/src/components/AssistantChat/UserEditComposer.tsx new file mode 100644 index 0000000000..8673fc9662 --- /dev/null +++ b/web/packages/common/src/components/AssistantChat/UserEditComposer.tsx @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ComposerPrimitive, MessagePrimitive } from '@assistant-ui/react'; +import { ComposerAttachmentsRow } from '@nemo/common/src/components/AssistantChat/ComposerAttachments'; +import { Button, Flex, Text, TextArea, Tooltip } from '@nvidia/foundations-react-core'; +import { ImagePlus, X } from 'lucide-react'; + +export const UserEditComposer = ({ + enableImageAttachments = true, +}: { + enableImageAttachments?: boolean; +}) => ( + + + {enableImageAttachments && } + } + /> + + {enableImageAttachments && ( + + + + + + )} + + + + + + + + + + + +); diff --git a/web/packages/common/src/components/AssistantChat/UserMessage.tsx b/web/packages/common/src/components/AssistantChat/UserMessage.tsx new file mode 100644 index 0000000000..3d1bdb5ca1 --- /dev/null +++ b/web/packages/common/src/components/AssistantChat/UserMessage.tsx @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ActionBarPrimitive, MessagePrimitive } from '@assistant-ui/react'; +import { AssistantChatMessageContent } from '@nemo/common/src/components/AssistantChat/AssistantChatMessageContent'; +import { + ACTION_BUTTON_CLASS, + CopyAction, + MESSAGE_ACTIONS_CLASS, +} from '@nemo/common/src/components/AssistantChat/messageActions'; +import type { MessageRenderProps } from '@nemo/common/src/components/AssistantChat/types'; +import { Tooltip } from '@nvidia/foundations-react-core'; +import { Pencil } from 'lucide-react'; + +export const UserMessage = ({ messageContentProps, toolCallPartComponent }: MessageRenderProps) => ( + +
+ +
+
+ + + + + + + + +
+
+); diff --git a/web/packages/common/src/components/AssistantChat/index.test.tsx b/web/packages/common/src/components/AssistantChat/index.test.tsx index 6d1af6d483..c4d553b425 100644 --- a/web/packages/common/src/components/AssistantChat/index.test.tsx +++ b/web/packages/common/src/components/AssistantChat/index.test.tsx @@ -254,6 +254,101 @@ describe('AssistantChat', () => { expect(screen.getByRole('button', { name: /Edit message/i })).toBeInTheDocument(); }); + it('offers an enabled add-image affordance for a vision model when enabled', () => { + renderAssistantChat( + + ); + + const addImageButton = screen.getByRole('button', { name: /Add image/i }); + expect(addImageButton).toBeInTheDocument(); + expect(addImageButton).toBeEnabled(); + }); + + it('hides the add-image affordance when image attachments are disabled', () => { + renderAssistantChat( + + ); + + expect(screen.queryByRole('button', { name: /Add image/i })).not.toBeInTheDocument(); + }); + + it('hides the add-image affordance for a non-vision model even when enabled', () => { + renderAssistantChat( + + ); + + expect(screen.queryByRole('button', { name: /Add image/i })).not.toBeInTheDocument(); + }); + + it('disables the add-image affordance when the chat is disabled', () => { + renderAssistantChat( + + ); + + expect(screen.getByRole('button', { name: /Add image/i })).toBeDisabled(); + }); + + it('renders image content parts in a user message', () => { + const imageUrl = 'data:image/png;base64,AAAA'; + renderAssistantChat( + + ); + + expect(screen.getByText('Look at this')).toBeInTheDocument(); + const image = screen.getByRole('img', { name: /Attached image/i }); + expect(image).toHaveAttribute('src', imageUrl); + }); + + it('seeds the edit composer with the original image attachment', async () => { + const imageUrl = 'data:image/png;base64,AAAA'; + renderAssistantChat( + + ); + + await userEvent.click(screen.getByRole('button', { name: /Edit message/i })); + + const editComposer = screen.getByTestId('assistant-chat-edit-composer'); + expect(within(editComposer).getByRole('textbox', { name: /Edit message/i })).toHaveValue( + 'Look at this' + ); + expect(within(editComposer).getByRole('img')).toHaveAttribute('src', imageUrl); + expect(within(editComposer).getByRole('button', { name: /Remove/i })).toBeInTheDocument(); + }); + it('renders composer override content in place of the prompt input', () => { renderAssistantChat( { it( 'edits a user message and re-runs inference with the edited prompt', async () => { + const user = userEvent.setup({ delay: null }); mocks.createChatCompletion .mockResolvedValueOnce(createCompletion('Original response.')) .mockResolvedValueOnce(createCompletion('Edited response.')); renderAssistantChat(); - await userEvent.type( - screen.getByRole('textbox', { name: /Task prompt/i }), - 'Original prompt' - ); - await userEvent.click(screen.getByRole('button', { name: /Submit/i })); + await user.type(screen.getByRole('textbox', { name: /Task prompt/i }), 'Original prompt'); + await user.click(screen.getByRole('button', { name: /Submit/i })); expect(await screen.findByText('Original response.')).toBeInTheDocument(); - await userEvent.click(screen.getByRole('button', { name: /Edit message/i })); + await user.click(screen.getByRole('button', { name: /Edit message/i })); const editInput = screen.getByRole('textbox', { name: /Edit message/i }); expect(editInput).toHaveValue('Original prompt'); expect(editInput.tagName).toBe('TEXTAREA'); - await userEvent.clear(editInput); - await userEvent.type(editInput, 'Edited prompt'); - await userEvent.click(screen.getByRole('button', { name: /Save edit/i })); + await user.clear(editInput); + await user.type(editInput, 'Edited prompt'); + await user.click(screen.getByRole('button', { name: /Save edit/i })); expect(await screen.findByText('Edited response.')).toBeInTheDocument(); expect(screen.getByText('Edited prompt')).toBeInTheDocument(); @@ -363,6 +456,7 @@ describe('AssistantChat', () => { it( 'aborts a pending completion request when stop is clicked', async () => { + const user = userEvent.setup({ delay: null }); let requestSignal: AbortSignal | undefined; const abortError = new Error('aborted'); abortError.name = 'AbortError'; @@ -377,16 +471,14 @@ describe('AssistantChat', () => { renderAssistantChat(); - await userEvent.type( - screen.getByRole('textbox', { name: /Task prompt/i }), - 'Hang before stream' - ); - await userEvent.click(screen.getByRole('button', { name: /Submit/i })); + await user.type(screen.getByRole('textbox', { name: /Task prompt/i }), 'Hang before stream'); + await user.click(screen.getByRole('button', { name: /Submit/i })); await waitFor(() => expect(requestSignal).toBeDefined()); expect(requestSignal?.aborted).toBe(false); - await userEvent.click(screen.getByRole('button', { name: /Stop/i })); + // Wait for isRunning to be reflected in the DOM before clicking Stop. + await user.click(await screen.findByRole('button', { name: /Stop/i })); await waitFor(() => expect(requestSignal?.aborted).toBe(true)); await waitFor(() => diff --git a/web/packages/common/src/components/AssistantChat/index.tsx b/web/packages/common/src/components/AssistantChat/index.tsx index f80729bea7..e1c61fb311 100644 --- a/web/packages/common/src/components/AssistantChat/index.tsx +++ b/web/packages/common/src/components/AssistantChat/index.tsx @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { AssistantRuntimeProvider } from '@assistant-ui/react'; +import { modelSupportsImageAttachments } from '@nemo/common/src/components/AssistantChat/messageUtils'; import cn from 'classnames'; import { type FC, useMemo } from 'react'; @@ -35,7 +36,12 @@ export const AssistantChat: FC = ({ slotComposerStart, emptyState, composerOverride, + enableImageAttachments = true, }) => { + // Gate image attachments on both the caller's opt-in and a naive model + // capability check, so a text-only model never offers an image affordance. + const imageAttachmentsEnabled = enableImageAttachments && modelSupportsImageAttachments(model); + const { handleReset, runtime } = useAssistantChatRuntime({ model, workspace, @@ -50,6 +56,7 @@ export const AssistantChat: FC = ({ onEmptyChange, broadcast, stopCount, + enableImageAttachments: imageAttachmentsEnabled, }); const composerPlaceholder = useMemo( @@ -70,6 +77,7 @@ export const AssistantChat: FC = ({ slotComposerStart={slotComposerStart} emptyState={emptyState} composerOverride={composerOverride} + enableImageAttachments={imageAttachmentsEnabled} /> diff --git a/web/packages/common/src/components/AssistantChat/messageActions.tsx b/web/packages/common/src/components/AssistantChat/messageActions.tsx new file mode 100644 index 0000000000..446212d3e6 --- /dev/null +++ b/web/packages/common/src/components/AssistantChat/messageActions.tsx @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ActionBarPrimitive, MessagePrimitive } from '@assistant-ui/react'; +import { Tooltip } from '@nvidia/foundations-react-core'; +import { Check, Copy } from 'lucide-react'; + +export const ACTION_BUTTON_CLASS = + 'flex cursor-pointer size-8 items-center justify-center rounded text-base bg-surface-raised hover:bg-surface-sunken disabled:cursor-not-allowed disabled:opacity-50'; + +export const MESSAGE_ACTIONS_CLASS = + 'flex gap-density-xs opacity-0 transition-opacity group-hover/message:opacity-100 group-focus-within/message:opacity-100 [@media(hover:none)]:opacity-100'; + +export const CopyAction = () => ( + + + + + + + + + + +); diff --git a/web/packages/common/src/components/AssistantChat/messageUtils.test.ts b/web/packages/common/src/components/AssistantChat/messageUtils.test.ts index ec299ce844..e4880c4dbc 100644 --- a/web/packages/common/src/components/AssistantChat/messageUtils.test.ts +++ b/web/packages/common/src/components/AssistantChat/messageUtils.test.ts @@ -1,9 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ThreadMessageLike } from '@assistant-ui/react'; - -import { getOpenAIMessages } from './messageUtils'; +import type { AppendMessage, ThreadMessageLike } from '@assistant-ui/react'; +import { + getOpenAIMessages, + getUserMessageContent, + modelSupportsImageAttachments, +} from '@nemo/common/src/components/AssistantChat/messageUtils'; const createMessage = (role: ThreadMessageLike['role'], text: string): ThreadMessageLike => ({ id: `${role}-${text}`, @@ -11,6 +14,8 @@ const createMessage = (role: ThreadMessageLike['role'], text: string): ThreadMes content: [{ type: 'text', text }], }); +const DATA_URL = 'data:image/png;base64,AAAA'; + describe('getOpenAIMessages', () => { it('filters empty content for all OpenAI message roles', () => { const messages = [ @@ -34,4 +39,82 @@ describe('getOpenAIMessages', () => { { role: 'user', content: 'Hello' }, ]); }); + + it('forwards user images as OpenAI multimodal content alongside the text', () => { + const messages: ThreadMessageLike[] = [ + { + id: 'user-with-image', + role: 'user', + content: [ + { type: 'text', text: 'What is in this image?' }, + { type: 'image', image: DATA_URL }, + ], + }, + ]; + + expect(getOpenAIMessages(messages)).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'What is in this image?' }, + { type: 'image_url', image_url: { url: DATA_URL } }, + ], + }, + ]); + }); + + it('sends an image-only user message without a text part', () => { + const messages: ThreadMessageLike[] = [ + { id: 'image-only', role: 'user', content: [{ type: 'image', image: DATA_URL }] }, + ]; + + expect(getOpenAIMessages(messages)).toEqual([ + { role: 'user', content: [{ type: 'image_url', image_url: { url: DATA_URL } }] }, + ]); + }); +}); + +describe('getUserMessageContent', () => { + it('combines composed text with image parts from attachments', () => { + const message = { + role: 'user', + content: [{ type: 'text', text: ' Describe this ' }], + attachments: [ + { + id: 'a1', + type: 'image', + name: 'pic.png', + status: { type: 'complete' }, + content: [{ type: 'image', image: DATA_URL }], + }, + ], + } as unknown as AppendMessage; + + expect(getUserMessageContent(message)).toEqual([ + { type: 'text', text: 'Describe this' }, + { type: 'image', image: DATA_URL }, + ]); + }); + + it('returns an empty array when there is neither text nor an attachment', () => { + const message = { + role: 'user', + content: [{ type: 'text', text: ' ' }], + } as unknown as AppendMessage; + + expect(getUserMessageContent(message)).toEqual([]); + }); +}); + +describe('modelSupportsImageAttachments', () => { + it('matches the vision substring case-insensitively', () => { + expect(modelSupportsImageAttachments('meta/llama-3.2-11b-vision-instruct')).toBe(true); + expect(modelSupportsImageAttachments('Some-Vision-Model')).toBe(true); + }); + + it('rejects models without the vision substring or with no name', () => { + expect(modelSupportsImageAttachments('meta/llama-3.1-8b-instruct')).toBe(false); + expect(modelSupportsImageAttachments('')).toBe(false); + expect(modelSupportsImageAttachments(undefined)).toBe(false); + }); }); diff --git a/web/packages/common/src/components/AssistantChat/messageUtils.ts b/web/packages/common/src/components/AssistantChat/messageUtils.ts index 5e00e86329..aac86acc66 100644 --- a/web/packages/common/src/components/AssistantChat/messageUtils.ts +++ b/web/packages/common/src/components/AssistantChat/messageUtils.ts @@ -3,11 +3,22 @@ import type { AppendMessage, + ImageMessagePart, MessageStatus, TextMessagePart, ThreadMessageLike, } from '@assistant-ui/react'; -import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'; +import type { + ChatCompletionContentPart, + ChatCompletionMessageParam, +} from 'openai/resources/index.mjs'; + +/** Content parts a user message can carry into the runtime: text plus images. */ +export type UserMessageContentPart = TextMessagePart | ImageMessagePart; + +const supportsImagesSubstrings = ['vision', 'image', 'images']; +export const modelSupportsImageAttachments = (model?: string): boolean => + !!model && supportsImagesSubstrings.some((substring) => model.toLowerCase().includes(substring)); const createMessageId = (): string => { if (typeof crypto !== 'undefined' && crypto.randomUUID) { @@ -32,6 +43,16 @@ const isTextPart = (part: unknown): part is TextMessagePart => { return 'type' in part && part.type === 'text' && 'text' in part && typeof part.text === 'string'; }; +const isImagePart = (part: unknown): part is ImageMessagePart => { + if (typeof part !== 'object' || part === null) return false; + return ( + 'type' in part && + part.type === 'image' && + 'image' in part && + typeof (part as ImageMessagePart).image === 'string' + ); +}; + export const getMessageText = (message: Pick): string => { if (typeof message.content === 'string') return message.content; return message.content @@ -40,6 +61,39 @@ export const getMessageText = (message: Pick): str .join('\n'); }; +const collectImageParts = (parts: readonly unknown[]): ImageMessagePart[] => + parts.filter(isImagePart); + +/** + * Collects image parts from a message — both inline content parts and any + * composer attachments. The `SimpleImageAttachmentAdapter` emits an image part + * (a data URL) in each attachment's content on send, so the two sources are + * unified here. + */ +export const getMessageImageParts = (message: { + content: ThreadMessageLike['content']; + attachments?: readonly { readonly content: readonly unknown[] }[]; +}): ImageMessagePart[] => { + const fromContent = Array.isArray(message.content) ? collectImageParts(message.content) : []; + const fromAttachments = (message.attachments ?? []).flatMap((attachment) => + collectImageParts(attachment.content) + ); + return [...fromContent, ...fromAttachments]; +}; + +/** + * Builds the stored thread-message content for a freshly composed user message: + * the trimmed text (when present) followed by any image parts pulled from + * composer attachments. + */ +export const getUserMessageContent = (message: AppendMessage): UserMessageContentPart[] => { + const content: UserMessageContentPart[] = []; + const text = getMessageText(message).trim(); + if (text) content.push({ type: 'text', text }); + content.push(...getMessageImageParts(message)); + return content; +}; + export const appendMessageToThreadMessage = (message: AppendMessage): ThreadMessageLike => ({ id: createMessageId(), role: message.role, @@ -49,12 +103,25 @@ export const appendMessageToThreadMessage = (message: AppendMessage): ThreadMess const getOpenAIMessage = (message: ThreadMessageLike): ChatCompletionMessageParam | undefined => { if (!['assistant', 'system', 'user'].includes(message.role)) return undefined; - const content = getMessageText(message); - if (!content) return undefined; + const text = getMessageText(message); + + if (message.role === 'user') { + const imageParts = getMessageImageParts(message); + if (imageParts.length > 0) { + const content: ChatCompletionContentPart[] = []; + if (text) content.push({ type: 'text', text }); + for (const part of imageParts) { + content.push({ type: 'image_url', image_url: { url: part.image } }); + } + return { role: 'user', content }; + } + } + + if (!text) return undefined; return { role: message.role, - content, + content: text, } as ChatCompletionMessageParam; }; diff --git a/web/packages/common/src/components/AssistantChat/types.ts b/web/packages/common/src/components/AssistantChat/types.ts index 62f438f60b..3ec8fe9c5a 100644 --- a/web/packages/common/src/components/AssistantChat/types.ts +++ b/web/packages/common/src/components/AssistantChat/types.ts @@ -1,12 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ThreadMessageLike } from '@assistant-ui/react'; +import type { + ThreadMessageLike, + ThreadPrimitive, + ToolCallMessagePartComponent, +} from '@assistant-ui/react'; +import type { MessageContentProps } from '@nemo/common/src/components/Chat/MessageContent'; import type { PromptData } from '@nemo/sdk/generated/platform/schema'; import type { ChatCompletionTool } from 'openai/resources/index.mjs'; -import type { ReactNode } from 'react'; - -import type { AssistantChatThreadAttributes } from './AssistantChatThread'; +import type { ComponentProps, ReactNode } from 'react'; export const ComposerMode = { PER_PANEL: 'per-panel', @@ -14,6 +17,39 @@ export const ComposerMode = { } as const; export type ComposerMode = (typeof ComposerMode)[keyof typeof ComposerMode]; +export interface AssistantChatThreadAttributes { + ThreadViewport?: ComponentProps; +} + +export type AssistantChatMessageContentProps = Pick; + +export interface MessageRenderProps { + messageContentProps?: AssistantChatMessageContentProps; + toolCallPartComponent?: ToolCallMessagePartComponent; +} + +export interface AssistantChatThreadProps { + disabled?: boolean; + placeholder: string; + onReset: () => void; + showRunningIndicator?: boolean; + attributes?: AssistantChatThreadAttributes; + composerMode?: ComposerMode; + slotComposerStart?: ReactNode; + emptyState?: { + slotHeading?: string; + slotSubheading?: string; + }; + contentClassName?: string; + composerContainerClassName?: string; + hideAssistantMessageActions?: boolean; + toolCallPartComponent?: ToolCallMessagePartComponent; + viewportClassName?: string; + composerOverride?: ReactNode; + messageContentProps?: AssistantChatMessageContentProps; + enableImageAttachments?: boolean; +} + export interface AssistantChatProps { /** * The model name to route through inference gateway. @@ -95,6 +131,10 @@ export interface AssistantChatProps { slotSubheading?: string; }; composerOverride?: ReactNode; + /** + * @default true + */ + enableImageAttachments?: boolean; } export interface BroadcastSignal { diff --git a/web/packages/common/src/components/AssistantChat/useAssistantChatRuntime.ts b/web/packages/common/src/components/AssistantChat/useAssistantChatRuntime.ts index da8836770b..574e391b8b 100644 --- a/web/packages/common/src/components/AssistantChat/useAssistantChatRuntime.ts +++ b/web/packages/common/src/components/AssistantChat/useAssistantChatRuntime.ts @@ -5,6 +5,7 @@ import { type AppendMessage, type MessageStatus, type ThreadMessageLike, + SimpleImageAttachmentAdapter, useExternalStoreRuntime, } from '@assistant-ui/react'; import { useCallback, useEffect, useRef, useState } from 'react'; @@ -15,12 +16,14 @@ import { appendMessageToThreadMessage, createTextMessage, getEditedMessageIndex, - getMessageText, getOpenAIMessages, + getUserMessageContent, } from './messageUtils'; import type { AssistantChatProps } from './types'; import { useChatCompletion } from '../../hooks/useChatCompletion'; +const imageAttachmentAdapter = new SimpleImageAttachmentAdapter(); + type UseAssistantChatRuntimeOptions = Pick< AssistantChatProps, | 'baseURL' @@ -36,6 +39,7 @@ type UseAssistantChatRuntimeOptions = Pick< | 'promptData' | 'tools' | 'workspace' + | 'enableImageAttachments' >; export const useAssistantChatRuntime = ({ @@ -52,6 +56,7 @@ export const useAssistantChatRuntime = ({ onEmptyChange, broadcast, stopCount, + enableImageAttachments = true, }: UseAssistantChatRuntimeOptions) => { const [messages, setMessages] = useState(initialMessages); const [isRunning, setIsRunning] = useState(false); @@ -216,13 +221,10 @@ export const useAssistantChatRuntime = ({ const handleNewMessage = useCallback( async (message: AppendMessage) => { - const text = getMessageText(message).trim(); - if (!text) return; + const content = getUserMessageContent(message); + if (content.length === 0) return; - const userMessage = appendMessageToThreadMessage({ - ...message, - content: [{ type: 'text', text }], - }); + const userMessage = appendMessageToThreadMessage({ ...message, content }); const nextMessages = [...messagesRef.current, userMessage]; setThreadMessages(nextMessages); await runCompletion(nextMessages); @@ -248,15 +250,12 @@ export const useAssistantChatRuntime = ({ const sourceIndex = getEditedMessageIndex(messagesRef.current, message); const previousMessages = sourceIndex === -1 ? messagesRef.current : messagesRef.current.slice(0, sourceIndex); - const text = getMessageText(message).trim(); - if (!text) return; + const content = getUserMessageContent(message); + if (content.length === 0) return; const nextMessages = [ ...previousMessages, - appendMessageToThreadMessage({ - ...message, - content: [{ type: 'text', text }], - }), + appendMessageToThreadMessage({ ...message, content }), ]; setThreadMessages(nextMessages); await runCompletion(nextMessages); @@ -340,6 +339,7 @@ export const useAssistantChatRuntime = ({ onReload: async () => handleReload(), onCancel: handleCancel, convertMessage: (message) => message, + adapters: enableImageAttachments ? { attachments: imageAttachmentAdapter } : undefined, unstable_capabilities: { copy: true, },