diff --git a/changelogs/fragments/11134.yml b/changelogs/fragments/11134.yml new file mode 100644 index 000000000000..7db2a991bb04 --- /dev/null +++ b/changelogs/fragments/11134.yml @@ -0,0 +1,3 @@ +feat: +- Add context awareness for explore visualizations ([#11134](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/11134)) +- Add `Ask AI` Context Menu Action to explore visualizations ([#11134](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/11134)) \ No newline at end of file diff --git a/package.json b/package.json index 00980be84cd0..201102a58771 100644 --- a/package.json +++ b/package.json @@ -268,6 +268,7 @@ "globby": "^11.1.0", "handlebars": "4.7.7", "hjson": "3.2.1", + "html2canvas": "^1.4.1", "http-aws-es": "npm:@zhongnansu/http-aws-es@6.0.1", "http-proxy-agent": "^2.1.0", "https-proxy-agent": "^5.0.0", diff --git a/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts b/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts index 97774d1877ef..e017b734d27a 100644 --- a/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts +++ b/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts @@ -462,17 +462,58 @@ export class ReactGraphNodes { } return true; }) - .map((msg) => ({ + .map((msg) => { // Convert 'tool' role to 'user' role for Bedrock compatibility // Bedrock only accepts 'user' and 'assistant' roles - role: msg.role === 'tool' ? 'user' : msg.role || 'user', - // If content is already an array (proper format), use it directly - // This preserves toolUse and toolResult blocks - // Filter out empty text blocks to prevent ValidationException - content: Array.isArray(msg.content) - ? msg.content.filter((block: any) => !block.text || block.text.trim() !== '') - : [{ text: msg.content || '' }].filter((block: any) => block.text.trim() !== ''), - })); + const role = msg.role === 'tool' ? 'user' : msg.role || 'user'; + + // Process content array to handle binary/image content + let content: any[]; + if (Array.isArray(msg.content)) { + content = msg.content + .map((block: any) => { + // Handle binary content (images) + if (block.type === 'binary' && block.data) { + // Convert binary content to Bedrock image format + // The AWS SDK expects image data as Uint8Array (bytes), not base64 string + // Extract format from mimeType (e.g., 'image/jpeg' -> 'jpeg') + const format = block.mimeType?.includes('/') + ? block.mimeType.split('/')[1] + : block.mimeType || 'jpeg'; + + // Convert base64 string to Uint8Array for AWS SDK + const imageBytes = Buffer.from(block.data, 'base64'); + + return { + image: { + format, + source: { + bytes: imageBytes, + }, + }, + }; + } + // Handle text content blocks - extract just the text field for Bedrock + if (block.type === 'text' && block.text) { + return { text: block.text }; + } + // Keep other content blocks as-is (toolUse, toolResult) + return block; + }) + .filter((block: any) => { + // Filter out empty text blocks to prevent ValidationException + if (block.text !== undefined) { + return block.text.trim() !== ''; + } + return true; + }); + } else { + // Handle string content + content = [{ text: msg.content || '' }].filter((block: any) => block.text.trim() !== ''); + } + + return { role, content }; + }); // Debug logging to catch toolUse/toolResult mismatch let toolUseCount = 0; diff --git a/src/core/public/chat/types.ts b/src/core/public/chat/types.ts index e7a58bc5e3f6..2a8978753cfc 100644 --- a/src/core/public/chat/types.ts +++ b/src/core/public/chat/types.ts @@ -5,6 +5,22 @@ import { Observable } from 'rxjs'; +interface TextInputContent { + type: 'text'; + text: string; +} + +interface BinaryInputContent { + type: 'binary'; + mimeType: string; + id?: string; + url?: string; + data?: string; + filename?: string; +} + +type InputContent = TextInputContent | BinaryInputContent; + /** * Function call interface */ @@ -28,7 +44,7 @@ export interface ToolCall { export interface BaseMessage { id: string; role: string; - content?: string; + content?: string | InputContent[]; name?: string; } @@ -62,7 +78,7 @@ export interface AssistantMessage extends BaseMessage { */ export interface UserMessage extends BaseMessage { role: 'user'; - content: string; + content: string | InputContent[]; } /** diff --git a/src/core/public/mocks.ts b/src/core/public/mocks.ts index 7cfbe886d9da..b045033b984e 100644 --- a/src/core/public/mocks.ts +++ b/src/core/public/mocks.ts @@ -94,6 +94,7 @@ function createCoreSetupMock({ }, workspaces: workspacesServiceMock.createSetupContract(), keyboardShortcut: keyboardShortcutServiceMock.createSetup(), + chat: coreChatServiceMock.createSetupContract(), }; return mock; diff --git a/src/plugins/chat/common/types.ts b/src/plugins/chat/common/types.ts index d7eb4a636e25..4e6ff57a0184 100644 --- a/src/plugins/chat/common/types.ts +++ b/src/plugins/chat/common/types.ts @@ -30,6 +30,23 @@ export const ToolCallSchema = z.object({ function: FunctionCallSchema, }); +// AG-UI Protocol: Input content schemas for multimodal content (text + images) +export const TextInputContentSchema = z.object({ + type: z.literal('text'), + text: z.string(), +}); + +export const BinaryInputContentSchema = z.object({ + type: z.literal('binary'), + mimeType: z.string(), + id: z.string().optional(), + url: z.string().optional(), + data: z.string().optional(), + filename: z.string().optional(), +}); + +export const InputContentSchema = z.union([TextInputContentSchema, BinaryInputContentSchema]); + export const BaseMessageSchema = z.object({ id: z.string(), role: z.string(), @@ -55,7 +72,7 @@ export const AssistantMessageSchema = BaseMessageSchema.extend({ export const UserMessageSchema = BaseMessageSchema.extend({ role: z.literal('user'), - content: z.string(), + content: z.union([z.string(), z.array(InputContentSchema)]), }); export const ToolMessageSchema = z.object({ diff --git a/src/plugins/chat/public/components/chat_window.tsx b/src/plugins/chat/public/components/chat_window.tsx index a5d261a3014c..5914fbc651d2 100644 --- a/src/plugins/chat/public/components/chat_window.tsx +++ b/src/plugins/chat/public/components/chat_window.tsx @@ -30,7 +30,7 @@ import { ChatInput } from './chat_input'; export interface ChatWindowInstance{ startNewChat: ()=>void; - sendMessage: (options:{content: string})=>Promise; + sendMessage: (options:{content: string; messages?: Message[]})=>Promise; } interface ChatWindowProps { @@ -63,9 +63,9 @@ const ChatWindowContent = React.forwardRef( const [isStreaming, setIsStreaming] = useState(false); const [currentRunId, setCurrentRunId] = useState(null); const handleSendRef = useRef(); - + const timelineRef = React.useRef(timeline); - + React.useEffect(() => { timelineRef.current = timeline; }, [timeline]); @@ -115,7 +115,7 @@ const ChatWindowContent = React.forwardRef( chatService.updateCurrentMessages(timeline); }, [timeline, chatService]); - const handleSend = async (options?: {input?: string}) => { + const handleSend = async (options?: {input?: string; messages?: Message[]}) => { const messageContent = options?.input ?? input.trim(); if (!messageContent || isStreaming) return; @@ -123,12 +123,19 @@ const ChatWindowContent = React.forwardRef( setIsStreaming(true); try { + // Prepare additional messages for sending (but don't add to timeline yet) + const additionalMessages = options?.messages ?? []; + + // Merge additional messages with current timeline for sending + const messagesToSend = [...timeline, ...additionalMessages]; + const { observable, userMessage } = await chatService.sendMessage( messageContent, - timeline + messagesToSend ); - // Add user message immediately to timeline + // Add the final merged user message to timeline + // (chat_service already merged any additional messages with the text) const timelineUserMessage: UserMessage = { id: userMessage.id, role: 'user', @@ -185,6 +192,24 @@ const ChatWindowContent = React.forwardRef( if (messageIndex === -1) return; + let textContent = typeof message.content === "string" ? message.content : ""; + const additionalMessages: Message[] = []; + + if (Array.isArray(message.content)) { + const lastMessageContent = message.content[message.content.length - 1]; + if (lastMessageContent.type === "text") { + textContent = lastMessageContent.text; + additionalMessages.push({ + ...message, + content: message.content.slice(0, message.content.length - 1), + }); + } + } + + if (textContent === "") { + return; + } + // Remove this message and everything after it from the timeline const truncatedTimeline = timeline.slice(0, messageIndex); setTimeline(truncatedTimeline); @@ -195,8 +220,8 @@ const ChatWindowContent = React.forwardRef( try { const { observable, userMessage } = await chatService.sendMessage( - message.content, - truncatedTimeline + textContent, + [...truncatedTimeline,...additionalMessages] ); // Add user message immediately to timeline @@ -249,7 +274,7 @@ const ChatWindowContent = React.forwardRef( useImperativeHandle(ref, ()=>({ startNewChat: ()=>handleNewChat(), - sendMessage: async ({content})=>(await handleSendRef.current?.({input:content})) + sendMessage: async ({content, messages})=>(await handleSendRef.current?.({input:content, messages})) }), [handleNewChat]); return ( diff --git a/src/plugins/chat/public/components/message_row.test.tsx b/src/plugins/chat/public/components/message_row.test.tsx new file mode 100644 index 000000000000..1c14e2d8156f --- /dev/null +++ b/src/plugins/chat/public/components/message_row.test.tsx @@ -0,0 +1,238 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { MessageRow } from './message_row'; +import { Message } from '../../common/types'; + +// Mock the Markdown component +jest.mock('../../../opensearch_dashboards_react/public', () => ({ + Markdown: ({ markdown }: { markdown: string }) =>
{markdown}
, +})); + +describe('MessageRow', () => { + describe('string content rendering', () => { + it('should render simple text message', () => { + const message: Message = { + id: 'msg-1', + role: 'user', + content: 'Hello, world!', + }; + + render(); + + expect(screen.getByText('Hello, world!')).toBeInTheDocument(); + }); + + it('should render empty content gracefully', () => { + const message: Message = { + id: 'msg-2', + role: 'assistant', + content: '', + }; + + const { container } = render(); + + // Just check that the component renders without errors + expect(container.querySelector('.messageRow')).toBeInTheDocument(); + }); + }); + + describe('multimodal content rendering', () => { + it('should render image from binary content', () => { + const message: Message = { + id: 'msg-3', + role: 'user', + content: [ + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'base64encodedimagedata', + }, + ], + }; + + render(); + + const img = screen.getByRole('img'); + expect(img).toHaveAttribute('src', 'data:image/jpeg;base64,base64encodedimagedata'); + expect(img).toHaveAttribute('alt', 'Visualization'); + }); + + it('should render image with custom filename', () => { + const message: Message = { + id: 'msg-4', + role: 'user', + content: [ + { + type: 'binary', + mimeType: 'image/png', + data: 'imagedata', + filename: 'chart.png', + }, + ], + }; + + render(); + + const img = screen.getByRole('img'); + expect(img).toHaveAttribute('alt', 'chart.png'); + }); + + it('should render text block from array content', () => { + const message: Message = { + id: 'msg-5', + role: 'user', + content: [ + { + type: 'text', + text: 'Analyze this data', + }, + ], + }; + + render(); + + expect(screen.getByText('Analyze this data')).toBeInTheDocument(); + }); + + it('should render both image and text in correct order', () => { + const message: Message = { + id: 'msg-6', + role: 'user', + content: [ + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'imagedata', + }, + { + type: 'text', + text: 'What is in this image?', + }, + ], + }; + + const { container } = render(); + + // Check both image and text are rendered + const img = screen.getByRole('img'); + expect(img).toBeInTheDocument(); + expect(screen.getByText('What is in this image?')).toBeInTheDocument(); + + // Verify order: image should come before text + const messageContent = container.querySelector('.messageRow__markdown'); + const children = Array.from(messageContent?.children || []); + expect(children[0].tagName).toBe('IMG'); + }); + + it('should render multiple images', () => { + const message: Message = { + id: 'msg-7', + role: 'user', + content: [ + { + type: 'binary', + mimeType: 'image/png', + data: 'firstimage', + }, + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'secondimage', + }, + ], + }; + + render(); + + const images = screen.getAllByRole('img'); + expect(images).toHaveLength(2); + expect(images[0]).toHaveAttribute('src', 'data:image/png;base64,firstimage'); + expect(images[1]).toHaveAttribute('src', 'data:image/jpeg;base64,secondimage'); + }); + + it('should handle mixed content with backward compatibility', () => { + const message: Message = { + id: 'msg-8', + role: 'user', + content: [ + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'imagedata', + }, + { + type: 'text', + text: 'Plain text block', + }, + ], + }; + + render(); + + expect(screen.getByRole('img')).toBeInTheDocument(); + expect(screen.getByText('Plain text block')).toBeInTheDocument(); + }); + }); + + describe('role-based styling', () => { + it('should apply user styling for user messages', () => { + const message: Message = { + id: 'msg-11', + role: 'user', + content: 'User message', + }; + + const { container } = render(); + + const messageRow = container.querySelector('.messageRow'); + expect(messageRow).toHaveClass('messageRow--user'); + }); + + it('should not apply user styling for assistant messages', () => { + const message: Message = { + id: 'msg-12', + role: 'assistant', + content: 'Assistant message', + }; + + const { container } = render(); + + const messageRow = container.querySelector('.messageRow'); + expect(messageRow).not.toHaveClass('messageRow--user'); + }); + }); + + describe('streaming indicator', () => { + it('should show cursor when streaming', () => { + const message: Message = { + id: 'msg-13', + role: 'assistant', + content: 'Streaming...', + }; + + const { container } = render(); + + const cursor = container.querySelector('.messageRow__cursor'); + expect(cursor).toBeInTheDocument(); + expect(cursor).toHaveTextContent('|'); + }); + + it('should not show cursor when not streaming', () => { + const message: Message = { + id: 'msg-14', + role: 'assistant', + content: 'Complete message', + }; + + const { container } = render(); + + const cursor = container.querySelector('.messageRow__cursor'); + expect(cursor).not.toBeInTheDocument(); + }); + }); +}); diff --git a/src/plugins/chat/public/components/message_row.tsx b/src/plugins/chat/public/components/message_row.tsx index c8e60246e017..5f278cf7824a 100644 --- a/src/plugins/chat/public/components/message_row.tsx +++ b/src/plugins/chat/public/components/message_row.tsx @@ -28,8 +28,48 @@ export const MessageRow: React.FC = ({ } }; - // Handle optional content - const content = message.content || ''; + // Handle multimodal content (text + images) or simple string content + const renderContent = () => { + const content = message.content || ''; + + // If content is a string, render as markdown + if (typeof content === 'string') { + return ; + } + + // If content is an array, handle multimodal content (text + binary) + if (Array.isArray(content)) { + return ( + <> + {content.map((block: any, index: number) => { + // Render binary content (images) + if (block.type === 'binary' && block.data) { + return ( + {block.filename + ); + } + // Render text content as markdown + if (block.type === 'text' && block.text) { + return ; + } + // Handle plain text blocks (for backward compatibility) + if (block.text) { + return ; + } + return null; + })} + + ); + } + + // Fallback for any other type + return ; + }; return (
= ({
- + {renderContent()} {isStreaming && |}
diff --git a/src/plugins/chat/public/services/chat_service.test.ts b/src/plugins/chat/public/services/chat_service.test.ts index d042e36a5591..7e0295ce2522 100644 --- a/src/plugins/chat/public/services/chat_service.test.ts +++ b/src/plugins/chat/public/services/chat_service.test.ts @@ -350,6 +350,126 @@ describe('ChatService', () => { undefined ); // dataSourceId is undefined when no uiSettings provided }); + + it('should merge text with multimodal content when last message has array content', async () => { + const mockObservable = new Observable(); + mockAgent.runAgent.mockReturnValue(mockObservable); + + const imageMessage: Message = { + id: 'image-msg-1', + role: 'user', + content: [ + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'base64encodeddata', + }, + ], + }; + + const result = await chatService.sendMessage('Analyze this image', [imageMessage]); + + // Should merge image with text into a single message + expect(result.userMessage.content).toEqual([ + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'base64encodeddata', + }, + { + type: 'text', + text: 'Analyze this image', + }, + ]); + + // Should have a new ID + expect(result.userMessage.id).not.toBe('image-msg-1'); + expect(result.userMessage.id).toMatch(/^msg-\d+-[a-z0-9]{9}$/); + + // Should only send the merged message (not the original image message) + expect(mockAgent.runAgent).toHaveBeenCalledWith( + expect.objectContaining({ + messages: [result.userMessage], + }), + undefined + ); + }); + + it('should preserve order when merging multimodal content', async () => { + const mockObservable = new Observable(); + mockAgent.runAgent.mockReturnValue(mockObservable); + + const multimodalMessage: Message = { + id: 'multi-msg-1', + role: 'user', + content: [ + { + type: 'binary', + mimeType: 'image/png', + data: 'firstimage', + }, + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'secondimage', + }, + ], + }; + + const result = await chatService.sendMessage('What are these?', [multimodalMessage]); + + // Should preserve order: first image, second image, then text + expect(result.userMessage.content).toEqual([ + { + type: 'binary', + mimeType: 'image/png', + data: 'firstimage', + }, + { + type: 'binary', + mimeType: 'image/jpeg', + data: 'secondimage', + }, + { + type: 'text', + text: 'What are these?', + }, + ]); + }); + + it('should create simple text message when no array content exists', async () => { + const mockObservable = new Observable(); + mockAgent.runAgent.mockReturnValue(mockObservable); + + const textMessage: Message = { + id: 'text-msg-1', + role: 'user', + content: 'Previous message', + }; + + const result = await chatService.sendMessage('New message', [textMessage]); + + // Should create a simple text message (not array) + expect(result.userMessage.content).toBe('New message'); + expect(Array.isArray(result.userMessage.content)).toBe(false); + }); + + it('should not merge when last message is not a user message', async () => { + const mockObservable = new Observable(); + mockAgent.runAgent.mockReturnValue(mockObservable); + + const assistantMessage: Message = { + id: 'assistant-msg-1', + role: 'assistant', + content: 'I am the assistant', + }; + + const result = await chatService.sendMessage('User message', [assistantMessage]); + + // Should create a simple text message + expect(result.userMessage.content).toBe('User message'); + expect(Array.isArray(result.userMessage.content)).toBe(false); + }); }); describe('sendToolResult', () => { @@ -1082,7 +1202,7 @@ describe('ChatService', () => { const result = await chatService.sendMessageWithWindow('test message', []); - expect(mockSendMessage).toHaveBeenCalledWith({ content: 'test message' }); + expect(mockSendMessage).toHaveBeenCalledWith({ content: 'test message', messages: [] }); expect(result.userMessage.content).toBe('test message'); expect(result.observable).toBeDefined(); }); diff --git a/src/plugins/chat/public/services/chat_service.ts b/src/plugins/chat/public/services/chat_service.ts index 81f888090746..2446741d8601 100644 --- a/src/plugins/chat/public/services/chat_service.ts +++ b/src/plugins/chat/public/services/chat_service.ts @@ -250,7 +250,7 @@ export class ChatService { // If ChatWindow is available, delegate to its sendMessage for proper timeline management if (this.chatWindowRef?.current && this.isWindowOpen()) { try { - await this.chatWindowRef.current.sendMessage({ content }); + await this.chatWindowRef.current.sendMessage({ content, messages }); // Create a user message for consistency with the return type const userMessage: UserMessage = { @@ -361,11 +361,31 @@ export class ChatService { const requestId = this.generateRequestId(); this.addActiveRequest(requestId); - const userMessage: UserMessage = { - id: this.generateMessageId(), - role: 'user', - content: content.trim(), - }; + + // Check if the last message in the array is a user message with array content + // If so, append the text to the existing content array (for multimodal messages) + let userMessage: UserMessage; + const lastMessage = messages.length > 0 ? messages[messages.length - 1] : null; + const hasArrayContent = lastMessage?.role === 'user' && Array.isArray(lastMessage.content); + + if (hasArrayContent && lastMessage) { + // Remove the last message from the array since we'll merge it with the new message + messages = messages.slice(0, -1); + + // Append text to the existing content array (preserves order from caller) + userMessage = { + ...lastMessage, + id: this.generateMessageId(), + content: [...(lastMessage.content as any[]), { type: 'text', text: content.trim() }], + }; + } else { + // No array content, create a simple text message + userMessage = { + id: this.generateMessageId(), + role: 'user', + content: content.trim(), + }; + } // Get workspace-aware data source ID const dataSourceId = await this.getWorkspaceAwareDataSourceId(); diff --git a/src/plugins/explore/public/actions/ask_ai_embeddable_action.test.tsx b/src/plugins/explore/public/actions/ask_ai_embeddable_action.test.tsx new file mode 100644 index 000000000000..8eb2e6b2f18c --- /dev/null +++ b/src/plugins/explore/public/actions/ask_ai_embeddable_action.test.tsx @@ -0,0 +1,188 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, screen, waitFor } from '@testing-library/react'; +import { AskAIEmbeddableAction } from './ask_ai_embeddable_action'; +import { ExploreEmbeddable } from '../embeddable/explore_embeddable'; +import html2canvas from 'html2canvas'; + +// Mock html2canvas +jest.mock('html2canvas'); + +// Mock the loading overlay component +jest.mock('./ask_ai_embeddable_action', () => { + const actual = jest.requireActual('./ask_ai_embeddable_action'); + return { + ...actual, + LoadingOverlay: ({ onClose }: { onClose: () => void }) => ( +
+ +
+ ), + }; +}); + +describe('AskAIEmbeddableAction', () => { + let action: AskAIEmbeddableAction; + let mockCore: any; + let mockContextProvider: any; + let mockEmbeddable: any; + + beforeEach(() => { + // Mock core + mockCore = { + http: { + post: jest.fn(), + }, + notifications: { + toasts: { + addSuccess: jest.fn(), + addError: jest.fn(), + }, + }, + chat: { + isAvailable: () => true, + sendMessageWithWindow: jest.fn().mockResolvedValue(undefined), + }, + }; + + // Mock context provider + mockContextProvider = { + getAssistantContextStore: jest.fn().mockReturnValue({ + addContext: jest.fn().mockResolvedValue(undefined), + }), + }; + + // Mock embeddable + mockEmbeddable = ({ + id: 'test-embeddable-id', + getTitle: jest.fn().mockReturnValue('Test Visualization'), + type: 'explore', + getInput: jest.fn().mockReturnValue({ + savedObjectId: 'test-saved-object-id', + timeRange: { from: 'now-15m', to: 'now' }, + filters: [], + }), + savedExplore: { + searchSource: { + getFields: jest.fn().mockReturnValue({ + query: { + query: 'test query', + dataset: { + title: 'test-index', + dataSource: { + id: undefined, + }, + }, + }, + }), + }, + }, + node: document.createElement('div'), + } as unknown) as ExploreEmbeddable; + + // Create action instance + action = new AskAIEmbeddableAction(mockCore, mockContextProvider); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('getDisplayName', () => { + it('should return correct display name', () => { + expect(action.getDisplayName()).toBe('Ask AI'); + }); + }); + + describe('getIconType', () => { + it('should return correct icon type', () => { + expect(action.getIconType()).toBe('editorComment'); + }); + }); + + describe('isCompatible', () => { + it('should return true for ExploreEmbeddable', async () => { + const result = await action.isCompatible({ embeddable: mockEmbeddable }); + expect(result).toBe(true); + }); + + it('should return false for non-ExploreEmbeddable', async () => { + const nonExploreEmbeddable = { + type: 'other_type', + getInput: jest.fn(), + }; + const result = await action.isCompatible({ embeddable: nonExploreEmbeddable as any }); + expect(result).toBe(false); + }); + + it('should return false when context provider is not available', async () => { + const actionWithoutContext = new AskAIEmbeddableAction(mockCore, undefined); + const result = await actionWithoutContext.isCompatible({ embeddable: mockEmbeddable }); + expect(result).toBe(false); + }); + + it('should return false when chat is not available', async () => { + const actionWithoutContext = new AskAIEmbeddableAction( + { + ...mockCore, + chat: { + isAvailable: () => false, + }, + }, + undefined + ); + const result = await actionWithoutContext.isCompatible({ embeddable: mockEmbeddable }); + expect(result).toBe(false); + }); + }); + + describe('execute', () => { + beforeEach(() => { + // Mock DOM element + const mockElement = document.createElement('div'); + mockElement.className = 'embPanel__content'; + document.body.appendChild(mockElement); + + // Mock html2canvas + (html2canvas as jest.Mock).mockResolvedValue({ + toDataURL: jest.fn().mockReturnValue('data:image/png;base64,mockImageData'), + }); + + // Mock successful API response + mockCore.http.post.mockResolvedValue({ + summary: 'This is a test summary of the visualization', + }); + }); + + afterEach(() => { + document.body.innerHTML = ''; + }); + + it('should add context to context provider', async () => { + await action.execute({ embeddable: mockEmbeddable }); + + await waitFor(() => { + expect(mockContextProvider.getAssistantContextStore).toHaveBeenCalled(); + expect(mockContextProvider.getAssistantContextStore().addContext).toHaveBeenCalledWith( + expect.objectContaining({ + id: expect.stringContaining('visualization-'), + description: expect.stringContaining('Test Visualization'), + categories: ['visualization', 'dashboard', 'chat'], + }) + ); + }); + }); + + it('should call sendMessageWithWindow', async () => { + await action.execute({ embeddable: mockEmbeddable }); + + await waitFor(() => { + expect(mockCore.chat.sendMessageWithWindow).toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/src/plugins/explore/public/actions/ask_ai_embeddable_action.tsx b/src/plugins/explore/public/actions/ask_ai_embeddable_action.tsx new file mode 100644 index 000000000000..d5a4b95dee0a --- /dev/null +++ b/src/plugins/explore/public/actions/ask_ai_embeddable_action.tsx @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import { EuiIconType } from '@elastic/eui/src/components/icon/icon'; +import { get } from 'lodash'; +import html2canvas from 'html2canvas'; +import { EmbeddableContext, IEmbeddable } from '../../../embeddable/public'; +import { Action, IncompatibleActionError } from '../../../ui_actions/public'; +import { CoreStart } from '../../../../core/public'; +import { ContextProviderStart } from '../../../context_provider/public'; +import { SavedExplore } from '../saved_explore'; + +interface DiscoverVisualizationEmbeddable extends IEmbeddable { + savedExplore: SavedExplore; + node: HTMLElement; +} + +export const ASK_AI_EMBEDDABLE_ACTION = 'ASK_AI_EMBEDDABLE_ACTION'; + +// Extend the ActionContextMapping to include our action +declare module '../../../ui_actions/public' { + export interface ActionContextMapping { + [ASK_AI_EMBEDDABLE_ACTION]: EmbeddableContext; + } +} + +export class AskAIEmbeddableAction implements Action { + public readonly type = ASK_AI_EMBEDDABLE_ACTION; + public readonly id = ASK_AI_EMBEDDABLE_ACTION; + public order = 20; + + public grouping: Action['grouping'] = [ + { + id: ASK_AI_EMBEDDABLE_ACTION, + getDisplayName: () => this.getDisplayName(), + getIconType: () => this.getIconType(), + category: 'investigation', + order: 20, + }, + ]; + + constructor( + private readonly core: CoreStart, + private readonly contextProvider?: ContextProviderStart + ) {} + + public getIconType(): EuiIconType { + return 'editorComment'; + } + + public getDisplayName() { + return i18n.translate('explore.actions.askAIEmbeddable.displayName', { + defaultMessage: 'Ask AI', + }); + } + + public async isCompatible({ embeddable }: EmbeddableContext) { + // Check if this is an explore embeddable and if context provider is available + const hasContextProvider = this.contextProvider !== undefined; + return embeddable.type === 'explore' && hasContextProvider && this.core.chat.isAvailable(); + } + + public async execute({ embeddable }: EmbeddableContext) { + if (!(await this.isCompatible({ embeddable }))) { + throw new IncompatibleActionError(); + } + + const visEmbeddable = embeddable as DiscoverVisualizationEmbeddable; + + // Extract visualization context + const savedObjectId = get(visEmbeddable.getInput(), 'savedObjectId', ''); + const title = visEmbeddable.getTitle() || 'Untitled Visualization'; + const visType = visEmbeddable.type; + + // Get current filters, query, and time range + const input = visEmbeddable.getInput(); + const timeRange = input.timeRange; + const query = visEmbeddable.savedExplore.searchSource.getFields().query; + const filters = input.filters; + + try { + // Capture visualization as base64 image + let visualizationBase64 = ''; + + const canvas = await html2canvas(visEmbeddable.node, { + backgroundColor: '#ffffff', + logging: false, + useCORS: true, + }); + // Use JPEG format with low quality to save tokens + visualizationBase64 = canvas.toDataURL('image/jpeg', 0.5).split(',')[1]; + + // Create context for the assistant with summary + const visualizationContext = { + title, + visType, + savedObjectId, + timeRange, + query: query?.query, + filters, + index: query?.dataset?.title, + }; + + // Add context to the context provider + if (this.contextProvider) { + const contextStore = this.contextProvider.getAssistantContextStore(); + await contextStore.addContext({ + id: `visualization-${savedObjectId || embeddable.id}`, + description: `Visualization: ${title}`, + value: visualizationContext, + label: `Visualization: ${title}`, + categories: ['visualization', 'dashboard', 'chat'], + }); + } + + // Send visualization screenshot to chat + if (this.core.chat) { + // Create a message with the visualization image following AG-UI protocol + const imageMessage = { + role: 'user' as const, + id: `msg-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`, + content: [ + { + type: 'binary' as const, + mimeType: 'image/jpeg', + data: visualizationBase64, + }, + ], + }; + + // sendMessageWithWindow will open the chat window and send the message + await this.core.chat.sendMessageWithWindow( + 'Give me a summary for the selected visualization', + [imageMessage] + ); + } + } catch (error) { + this.core.notifications.toasts.addDanger({ + title: i18n.translate('explore.actions.askAIEmbeddable.errorTitle', { + defaultMessage: 'Failed to add visualization context', + }), + text: error instanceof Error ? error.message : String(error), + }); + } + } +} diff --git a/src/plugins/explore/public/plugin.test.ts b/src/plugins/explore/public/plugin.test.ts new file mode 100644 index 000000000000..95789141122a --- /dev/null +++ b/src/plugins/explore/public/plugin.test.ts @@ -0,0 +1,367 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ExplorePlugin } from './plugin'; +import { coreMock } from '../../../core/public/mocks'; +import { AskAIEmbeddableAction } from './actions/ask_ai_embeddable_action'; +import { CONTEXT_MENU_TRIGGER } from '../../embeddable/public'; +import { CoreSetup, CoreStart } from 'opensearch-dashboards/public'; +import { ExplorePluginStart, ExploreSetupDependencies, ExploreStartDependencies } from './types'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public'; +import { UrlForwardingSetup, UrlForwardingStart } from '../../url_forwarding/public'; +import { EmbeddableSetup, EmbeddableStart } from '../../embeddable/public'; +import { VisualizationsSetup, VisualizationsStart } from '../../visualizations/public'; +import { UiActionsSetup, UiActionsStart } from '../../ui_actions/public'; +import { NavigationPublicPluginStart as NavigationStart } from '../../navigation/public'; +import { + OpenSearchDashboardsLegacySetup, + OpenSearchDashboardsLegacyStart, +} from '../../opensearch_dashboards_legacy/public'; +import { UsageCollectionSetup } from '../../usage_collection/public'; +import { ExpressionsPublicPlugin, ExpressionsStart } from '../../expressions/public'; +import { DashboardSetup, DashboardStart } from '../../dashboard/public'; +import { ChartsPluginStart } from '../../charts/public'; +import { Start as InspectorPublicPluginStart } from '../../inspector/public'; +import { ContextProviderStart } from '../../context_provider/public'; + +// Mock the action +jest.mock('./actions/ask_ai_embeddable_action'); + +// Mock log action registry +jest.mock('./services/log_action_registry', () => ({ + logActionRegistry: { + registerAction: jest.fn(), + }, +})); + +// Mock createAskAiAction +jest.mock('./actions/ask_ai_action', () => ({ + createAskAiAction: jest.fn().mockReturnValue({ + id: 'ask_ai', + execute: jest.fn(), + }), +})); + +// Mock createOsdUrlTracker +jest.mock('../../opensearch_dashboards_utils/public', () => ({ + ...jest.requireActual('../../opensearch_dashboards_utils/public'), + createOsdUrlTracker: jest.fn(() => ({ + appMounted: jest.fn(), + appUnMounted: jest.fn(), + stop: jest.fn(), + })), +})); + +describe('ExplorePlugin', () => { + let plugin: ExplorePlugin; + let initializerContext: ReturnType; + let coreSetup: CoreSetup; + let coreStart: CoreStart; + let setupDeps: ExploreSetupDependencies; + let startDeps: ExploreStartDependencies; + + function createMockInitializerContext() { + return { + config: { + get: jest.fn().mockReturnValue({ + discoverTraces: { + enabled: false, + }, + }), + }, + logger: { + get: jest.fn().mockReturnValue({ + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }), + }, + env: { + packageInfo: { + version: '1.0.0', + }, + }, + }; + } + + function createMockSetupDeps(): ExploreSetupDependencies { + return { + data: ({ + __enhance: jest.fn(), + query: { + state$: { + pipe: jest.fn().mockReturnValue({ + subscribe: jest.fn(), + }), + }, + }, + } as unknown) as DataPublicPluginSetup, + urlForwarding: ({ + forwardApp: jest.fn(), + } as Partial) as UrlForwardingSetup, + embeddable: ({ + registerEmbeddableFactory: jest.fn(), + } as Partial) as EmbeddableSetup, + visualizations: ({ + registerAlias: jest.fn(), + all: jest.fn().mockReturnValue([]), + getAliases: jest.fn().mockReturnValue([]), + } as Partial) as VisualizationsSetup, + uiActions: ({ + getTriggerActions: jest.fn().mockReturnValue([]), + } as Partial) as UiActionsSetup, + navigation: {} as NavigationStart, + opensearchDashboardsLegacy: {} as OpenSearchDashboardsLegacySetup, + usageCollection: {} as UsageCollectionSetup, + expressions: {} as ReturnType, + dashboard: {} as DashboardSetup, + }; + } + + function createMockStartDeps(): ExploreStartDependencies { + return { + data: ({ + indexPatterns: {}, + dataViews: {}, + search: {}, + query: { + filterManager: {}, + timefilter: { + timefilter: {}, + }, + queryString: { + clearQuery: jest.fn(), + }, + }, + } as unknown) as DataPublicPluginStart, + uiActions: ({ + registerAction: jest.fn(), + addTriggerAction: jest.fn(), + detachAction: jest.fn(), + executeTriggerActions: jest.fn(), + registerTrigger: jest.fn(), + getTrigger: jest.fn(), + getTriggers: jest.fn(), + unregisterAction: jest.fn(), + attachAction: jest.fn(), + getAction: jest.fn(), + hasAction: jest.fn(), + } as Partial) as UiActionsStart, + dashboard: {} as DashboardStart, + expressions: ({ + ExpressionLoader: jest.fn(), + } as Partial) as ExpressionsStart, + charts: ({ + theme: {}, + } as Partial) as ChartsPluginStart, + navigation: {} as NavigationStart, + inspector: {} as InspectorPublicPluginStart, + urlForwarding: {} as UrlForwardingStart, + embeddable: {} as EmbeddableStart, + opensearchDashboardsLegacy: {} as OpenSearchDashboardsLegacyStart, + contextProvider: ({ + getAssistantContextStore: jest.fn().mockReturnValue({ + addContext: jest.fn(), + }), + } as Partial) as ContextProviderStart, + visualizations: ({ + all: jest.fn().mockReturnValue([]), + getAliases: jest.fn().mockReturnValue([]), + } as Partial) as VisualizationsStart, + }; + } + + beforeEach(() => { + // Mock initializer context + initializerContext = createMockInitializerContext(); + + // Mock core setup + coreSetup = coreMock.createSetup(); + coreSetup.getStartServices = jest.fn().mockResolvedValue([ + coreMock.createStart(), + { + data: { + indexPatterns: { + clearCache: jest.fn(), + }, + query: { + queryString: { + clearQuery: jest.fn(), + }, + }, + }, + uiActions: { + getTriggerActions: jest.fn().mockReturnValue([]), + }, + visualizations: { + all: jest.fn().mockReturnValue([]), + getAliases: jest.fn().mockReturnValue([]), + }, + }, + ]); + + // Mock core start + coreStart = coreMock.createStart(); + // Add workspaces mock with proper BehaviorSubject-like structure + Object.defineProperty(coreStart, 'workspaces', { + value: { + currentWorkspace$: { + pipe: jest.fn().mockReturnValue({ + toPromise: jest.fn().mockResolvedValue({ + features: ['observability'], + }), + }), + subscribe: jest.fn(), + getValue: jest.fn(), + next: jest.fn(), + }, + }, + writable: true, + configurable: true, + }); + + // Mock setup dependencies + setupDeps = createMockSetupDeps(); + + // Mock start dependencies + startDeps = createMockStartDeps(); + + plugin = new ExplorePlugin(initializerContext as any); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('setup', () => { + it('should register explore applications', () => { + plugin.setup(coreSetup as any, setupDeps as any); + + expect(coreSetup.application.register).toHaveBeenCalledTimes(4); + expect(coreSetup.application.register).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'explore/logs', + title: 'Logs', + }) + ); + expect(coreSetup.application.register).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'explore/traces', + title: 'Traces', + }) + ); + expect(coreSetup.application.register).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'explore/metrics', + title: 'Metrics', + }) + ); + expect(coreSetup.application.register).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'explore', + title: 'Discover', + }) + ); + }); + + it('should register embeddable factory', () => { + plugin.setup(coreSetup, setupDeps); + + expect(setupDeps.embeddable.registerEmbeddableFactory).toHaveBeenCalledWith( + 'explore', + expect.any(Object) + ); + }); + + it('should register visualization alias', () => { + plugin.setup(coreSetup, setupDeps); + + expect(setupDeps.visualizations.registerAlias).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'DiscoverVisualization', + aliasApp: 'explore', + title: expect.any(String), + }) + ); + }); + + it('should setup URL forwarding', () => { + plugin.setup(coreSetup, setupDeps); + + expect(setupDeps.urlForwarding.forwardApp).toHaveBeenCalledWith( + 'doc', + 'explore', + expect.any(Function) + ); + expect(setupDeps.urlForwarding.forwardApp).toHaveBeenCalledWith( + 'context', + 'explore', + expect.any(Function) + ); + expect(setupDeps.urlForwarding.forwardApp).toHaveBeenCalledWith( + 'discover', + 'explore', + expect.any(Function) + ); + }); + }); + + describe('start', () => { + beforeEach(() => { + plugin.setup(coreSetup, setupDeps); + }); + + it('should register Ask AI embeddable action when chat and contextProvider are available', () => { + plugin.start(coreStart, startDeps); + + expect(AskAIEmbeddableAction).toHaveBeenCalledWith(coreStart, startDeps.contextProvider); + expect(startDeps.uiActions.registerAction).toHaveBeenCalled(); + expect(startDeps.uiActions.addTriggerAction).toHaveBeenCalledWith( + CONTEXT_MENU_TRIGGER, + expect.any(Object) + ); + }); + + it('should not register Ask AI embeddable action when contextProvider is not available', () => { + const startDepsWithoutContextProvider = { + ...startDeps, + contextProvider: undefined, + }; + + plugin.start(coreStart, startDepsWithoutContextProvider); + + expect(AskAIEmbeddableAction).not.toHaveBeenCalled(); + expect(startDeps.uiActions.addTriggerAction).not.toHaveBeenCalledWith( + CONTEXT_MENU_TRIGGER, + expect.any(Object) + ); + }); + + it('should create saved explore loader', () => { + const result = plugin.start(coreStart, startDeps); + + expect(result.savedExploreLoader).toBeDefined(); + expect(result.savedSearchLoader).toBeDefined(); + expect(result.savedSearchLoader).toBe(result.savedExploreLoader); + }); + + it('should return visualization and slot registries', () => { + const result = plugin.start(coreStart, startDeps); + + expect(result.visualizationRegistry).toBeDefined(); + expect(result.slotRegistry).toBeDefined(); + }); + }); + + describe('stop', () => { + it('should call stop callbacks without errors', () => { + plugin.setup(coreSetup, setupDeps); + plugin.start(coreStart, startDeps); + + expect(() => plugin.stop()).not.toThrow(); + }); + }); +}); diff --git a/src/plugins/explore/public/plugin.ts b/src/plugins/explore/public/plugin.ts index 98054b013985..225af945e9d8 100644 --- a/src/plugins/explore/public/plugin.ts +++ b/src/plugins/explore/public/plugin.ts @@ -75,6 +75,8 @@ import { SlotRegistryService } from './services/slot_registry'; // Log Actions import { logActionRegistry } from './services/log_action_registry'; import { createAskAiAction } from './actions/ask_ai_action'; +import { AskAIEmbeddableAction } from './actions/ask_ai_embeddable_action'; +import { CONTEXT_MENU_TRIGGER } from '../../embeddable/public'; export class ExplorePlugin implements @@ -514,6 +516,12 @@ export class ExplorePlugin const askAiAction = createAskAiAction(core.chat); logActionRegistry.registerAction(askAiAction); + if (core.chat && plugins.contextProvider) { + const askAIEmbeddableAction = new AskAIEmbeddableAction(core, plugins.contextProvider); + plugins.uiActions.registerAction(askAIEmbeddableAction); + plugins.uiActions.addTriggerAction(CONTEXT_MENU_TRIGGER, askAIEmbeddableAction); + } + const savedExploreLoader = createSavedExploreLoader({ savedObjectsClient: core.savedObjects.client, indexPatterns: plugins.data.indexPatterns, diff --git a/yarn.lock b/yarn.lock index 3331c4d3500c..ca06b8e7318d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11303,6 +11303,11 @@ bare-stream@^2.0.0: dependencies: streamx "^2.21.0" +base64-arraybuffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc" + integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ== + base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -13093,6 +13098,13 @@ css-in-js-utils@^2.0.0: hyphenate-style-name "^1.0.2" isobject "^3.0.1" +css-line-break@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/css-line-break/-/css-line-break-2.1.0.tgz#bfef660dfa6f5397ea54116bb3cb4873edbc4fa0" + integrity sha512-FHcKFCZcAha3LwfVBhCQbW2nCNbkZXn7KVUJcsT5/P8YmfsVja0FMPJr0B903j/E69HUphKiV9iQArX8SDYA4w== + dependencies: + utrie "^1.0.2" + css-loader@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645" @@ -17747,6 +17759,14 @@ html-webpack-plugin@^5.0.0: pretty-error "^4.0.0" tapable "^2.0.0" +html2canvas@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/html2canvas/-/html2canvas-1.4.1.tgz#7cef1888311b5011d507794a066041b14669a543" + integrity sha512-fPU6BHNpsyIhr8yyMpTLLxAbkaK8ArIBcmZIRiBLiDhjeqvXolaEmDGmELFuX9I4xDcaKKcJl+TKZLqruBbmWA== + dependencies: + css-line-break "^2.1.0" + text-segmentation "^1.0.3" + htmlparser2@10.0.0: version "10.0.0" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-10.0.0.tgz#77ad249037b66bf8cc99c6e286ef73b83aeb621d" @@ -27543,6 +27563,13 @@ text-diff@^1.0.1: resolved "https://registry.yarnpkg.com/text-diff/-/text-diff-1.0.1.tgz#6c105905435e337857375c9d2f6ca63e453ff565" integrity sha1-bBBZBUNeM3hXN1ydL2ymPkU/9WU= +text-segmentation@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/text-segmentation/-/text-segmentation-1.0.3.tgz#52a388159efffe746b24a63ba311b6ac9f2d7943" + integrity sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw== + dependencies: + utrie "^1.0.2" + text-table@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" @@ -28684,6 +28711,13 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== +utrie@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/utrie/-/utrie-1.0.2.tgz#d42fe44de9bc0119c25de7f564a6ed1b2c87a645" + integrity sha512-1MLa5ouZiOmQzUbjbu9VmjLzn1QLXBhwpUa7kdLUQK+KQ5KA9I1vk5U4YHe/X2Ch7PYnJfWuWT+VbuxbGwljhw== + dependencies: + base64-arraybuffer "^1.0.2" + uuid-browser@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid-browser/-/uuid-browser-3.1.0.tgz#0f05a40aef74f9e5951e20efbf44b11871e56410"