feat: add context awareness for explore visualizations

changelogs/fragments/11134.yml

@@ -0,0 +1,3 @@
+feat:
+- Add context awareness for explore visualizations ([#11134](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/11134))
+- Add `Ask AI` Context Menu Action to explore visualizations ([#11134](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/11134))

package.json

@@ -268,6 +268,7 @@
     "globby": "^11.1.0",
     "handlebars": "4.7.7",
     "hjson": "3.2.1",
+    "html2canvas": "^1.4.1",
     "http-aws-es": "npm:@zhongnansu/http-aws-es@6.0.1",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^5.0.0",

packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts

@@ -462,17 +462,58 @@ export class ReactGraphNodes {
         }
         return true;
       })
-      .map((msg) => ({
+      .map((msg) => {
         // Convert 'tool' role to 'user' role for Bedrock compatibility
         // Bedrock only accepts 'user' and 'assistant' roles
-        role: msg.role === 'tool' ? 'user' : msg.role || 'user',
-        // If content is already an array (proper format), use it directly
-        // This preserves toolUse and toolResult blocks
-        // Filter out empty text blocks to prevent ValidationException
-        content: Array.isArray(msg.content)
-          ? msg.content.filter((block: any) => !block.text || block.text.trim() !== '')
-          : [{ text: msg.content || '' }].filter((block: any) => block.text.trim() !== ''),
-      }));
+        const role = msg.role === 'tool' ? 'user' : msg.role || 'user';
+
+        // Process content array to handle binary/image content
+        let content: any[];
+        if (Array.isArray(msg.content)) {
+          content = msg.content
+            .map((block: any) => {
+              // Handle binary content (images)
+              if (block.type === 'binary' && block.data) {
+                // Convert binary content to Bedrock image format
+                // The AWS SDK expects image data as Uint8Array (bytes), not base64 string
+                // Extract format from mimeType (e.g., 'image/jpeg' -> 'jpeg')
+                const format = block.mimeType?.includes('/')
+                  ? block.mimeType.split('/')[1]
+                  : block.mimeType || 'jpeg';
+
+                // Convert base64 string to Uint8Array for AWS SDK
+                const imageBytes = Buffer.from(block.data, 'base64');
+
+                return {
+                  image: {
+                    format,
+                    source: {
+                      bytes: imageBytes,
+                    },
+                  },
+                };
+              }
+              // Handle text content blocks - extract just the text field for Bedrock
+              if (block.type === 'text' && block.text) {
+                return { text: block.text };
+              }
+              // Keep other content blocks as-is (toolUse, toolResult)
+              return block;
+            })
+            .filter((block: any) => {
+              // Filter out empty text blocks to prevent ValidationException
+              if (block.text !== undefined) {
+                return block.text.trim() !== '';
+              }
+              return true;
+            });
+        } else {
+          // Handle string content
+          content = [{ text: msg.content || '' }].filter((block: any) => block.text.trim() !== '');
+        }
+
+        return { role, content };
+      });
 
     // Debug logging to catch toolUse/toolResult mismatch
     let toolUseCount = 0;

src/core/public/chat/types.ts

@@ -5,6 +5,22 @@
 
 import { Observable } from 'rxjs';
 
+interface TextInputContent {
+  type: 'text';
+  text: string;
+}
+
+interface BinaryInputContent {
+  type: 'binary';
+  mimeType: string;
+  id?: string;
+  url?: string;
+  data?: string;
+  filename?: string;
+}
+
+type InputContent = TextInputContent | BinaryInputContent;
+
 /**
  * Function call interface
  */
@@ -28,7 +44,7 @@ export interface ToolCall {
 export interface BaseMessage {
   id: string;
   role: string;
-  content?: string;
+  content?: string | InputContent[];
   name?: string;
 }
 
@@ -62,7 +78,7 @@ export interface AssistantMessage extends BaseMessage {
  */
 export interface UserMessage extends BaseMessage {
   role: 'user';
-  content: string;
+  content: string | InputContent[];
 }
 
 /**

src/core/public/mocks.ts

@@ -94,6 +94,7 @@ function createCoreSetupMock({
     },
     workspaces: workspacesServiceMock.createSetupContract(),
     keyboardShortcut: keyboardShortcutServiceMock.createSetup(),
+    chat: coreChatServiceMock.createSetupContract(),
   };
 
   return mock;

src/plugins/chat/common/types.ts

@@ -30,6 +30,23 @@ export const ToolCallSchema = z.object({
   function: FunctionCallSchema,
 });
 
+// AG-UI Protocol: Input content schemas for multimodal content (text + images)
+export const TextInputContentSchema = z.object({
+  type: z.literal('text'),
+  text: z.string(),
+});
+
+export const BinaryInputContentSchema = z.object({
+  type: z.literal('binary'),
+  mimeType: z.string(),
+  id: z.string().optional(),
+  url: z.string().optional(),
+  data: z.string().optional(),
+  filename: z.string().optional(),
+});
+
+export const InputContentSchema = z.union([TextInputContentSchema, BinaryInputContentSchema]);
+
 export const BaseMessageSchema = z.object({
   id: z.string(),
   role: z.string(),
@@ -55,7 +72,7 @@ export const AssistantMessageSchema = BaseMessageSchema.extend({
 
 export const UserMessageSchema = BaseMessageSchema.extend({
   role: z.literal('user'),
-  content: z.string(),
+  content: z.union([z.string(), z.array(InputContentSchema)]),
 });
 
 export const ToolMessageSchema = z.object({

src/plugins/chat/public/components/chat_window.tsx

@@ -30,7 +30,7 @@ import { ChatInput } from './chat_input';
 
 export interface ChatWindowInstance{
   startNewChat: ()=>void;
-  sendMessage: (options:{content: string})=>Promise<unknown>;
+  sendMessage: (options:{content: string; messages?: Message[]})=>Promise<unknown>;
 }
 
 interface ChatWindowProps {
@@ -63,9 +63,9 @@ const ChatWindowContent = React.forwardRef<ChatWindowInstance, ChatWindowProps>(
   const [isStreaming, setIsStreaming] = useState(false);
   const [currentRunId, setCurrentRunId] = useState<string | null>(null);
   const handleSendRef = useRef<typeof handleSend>();
-  
+
   const timelineRef = React.useRef<Message[]>(timeline);
-  
+
   React.useEffect(() => {
     timelineRef.current = timeline;
   }, [timeline]);
@@ -115,20 +115,27 @@ const ChatWindowContent = React.forwardRef<ChatWindowInstance, ChatWindowProps>(
     chatService.updateCurrentMessages(timeline);
   }, [timeline, chatService]);
 
-  const handleSend = async (options?: {input?: string}) => {
+  const handleSend = async (options?: {input?: string; messages?: Message[]}) => {
     const messageContent = options?.input ?? input.trim();
     if (!messageContent || isStreaming) return;
 
     setInput('');
     setIsStreaming(true);
 
     try {
+      // Prepare additional messages for sending (but don't add to timeline yet)
+      const additionalMessages = options?.messages ?? [];
+
+      // Merge additional messages with current timeline for sending
+      const messagesToSend = [...timeline, ...additionalMessages];
+
       const { observable, userMessage } = await chatService.sendMessage(
         messageContent,
-        timeline
+        messagesToSend
       );
 
-      // Add user message immediately to timeline
+      // Add the final merged user message to timeline
+      // (chat_service already merged any additional messages with the text)
       const timelineUserMessage: UserMessage = {
         id: userMessage.id,
         role: 'user',
@@ -185,6 +192,24 @@ const ChatWindowContent = React.forwardRef<ChatWindowInstance, ChatWindowProps>(
 
     if (messageIndex === -1) return;
 
+    let textContent = typeof message.content === "string" ? message.content : "";
+    const additionalMessages: Message[] = [];
+
+    if (Array.isArray(message.content)) {
+      const lastMessageContent =  message.content[message.content.length - 1];
+      if (lastMessageContent.type === "text") {
+        textContent = lastMessageContent.text;
+        additionalMessages.push({
+          ...message,
+          content: message.content.slice(0, message.content.length - 1),
+        });
+      }
+    }
+
+    if (textContent === "") {
+        return;
+    }
+
     // Remove this message and everything after it from the timeline
     const truncatedTimeline = timeline.slice(0, messageIndex);
     setTimeline(truncatedTimeline);
@@ -195,8 +220,8 @@ const ChatWindowContent = React.forwardRef<ChatWindowInstance, ChatWindowProps>(
 
     try {
       const { observable, userMessage } = await chatService.sendMessage(
-        message.content,
-        truncatedTimeline
+        textContent,
+        [...truncatedTimeline,...additionalMessages]
       );
 
       // Add user message immediately to timeline
@@ -249,7 +274,7 @@ const ChatWindowContent = React.forwardRef<ChatWindowInstance, ChatWindowProps>(
 
   useImperativeHandle(ref, ()=>({
     startNewChat: ()=>handleNewChat(),
-    sendMessage: async ({content})=>(await handleSendRef.current?.({input:content}))
+    sendMessage: async ({content, messages})=>(await handleSendRef.current?.({input:content, messages}))
   }), [handleNewChat]);
 
   return (