[8.x] [Obs AI Assistant] Avoid sending user prompts and LLM responses…

… to Telemetry (#208118) (#208606)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Obs AI Assistant] Avoid sending user prompts and LLM responses to
Telemetry (#208118)](#208118)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Viduni
Wickramarachchi","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-01-28T19:00:18Z","message":"[Obs
AI Assistant] Avoid sending user prompts and LLM responses to Telemetry
(#208118)\n\nCloses
https://github.com/elastic/obs-ai-assistant-team/issues/186\r\n\r\n###
Problem\r\nPrompts entered by users and responses from LLMs must not be
sent to\r\ntelemetry\r\n\r\n### Solution\r\nRemove `messages` from the
payloads sent to Telemetry\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] The PR
description includes the appropriate Release Notes section,\r\nand the
correct `release_note:*` label is applied per
the\r\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)","sha":"f10d8f7c56a63dc7e71ffcae8e31a5a1736da6ac","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Obs
AI
Assistant","ci:project-deploy-observability","backport:version","v8.18.0"],"title":"[Obs
AI Assistant] Avoid sending user prompts and LLM responses to
Telemetry","number":208118,"url":"https://github.com/elastic/kibana/pull/208118","mergeCommit":{"message":"[Obs
AI Assistant] Avoid sending user prompts and LLM responses to Telemetry
(#208118)\n\nCloses
https://github.com/elastic/obs-ai-assistant-team/issues/186\r\n\r\n###
Problem\r\nPrompts entered by users and responses from LLMs must not be
sent to\r\ntelemetry\r\n\r\n### Solution\r\nRemove `messages` from the
payloads sent to Telemetry\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] The PR
description includes the appropriate Release Notes section,\r\nand the
correct `release_note:*` label is applied per
the\r\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)","sha":"f10d8f7c56a63dc7e71ffcae8e31a5a1736da6ac"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/208118","number":208118,"mergeCommit":{"message":"[Obs
AI Assistant] Avoid sending user prompts and LLM responses to Telemetry
(#208118)\n\nCloses
https://github.com/elastic/obs-ai-assistant-team/issues/186\r\n\r\n###
Problem\r\nPrompts entered by users and responses from LLMs must not be
sent to\r\ntelemetry\r\n\r\n### Solution\r\nRemove `messages` from the
payloads sent to Telemetry\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] The PR
description includes the appropriate Release Notes section,\r\nand the
correct `release_note:*` label is applied per
the\r\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)","sha":"f10d8f7c56a63dc7e71ffcae8e31a5a1736da6ac"}},{"branch":"8.x","label":"v8.18.0","branchLabelMappingKey":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Viduni Wickramarachchi <[email protected]>

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_body.tsx

@@ -177,13 +177,24 @@ export function ChatBody({
 
   const [promptEditorHeight, setPromptEditorHeight] = useState<number>(0);
 
-  const handleFeedback = (message: Message, feedback: Feedback) => {
+  const handleFeedback = (feedback: Feedback) => {
     if (conversation.value?.conversation && 'user' in conversation.value) {
+      const {
+        messages: _removedMessages, // Exclude messages
+        conversation: { title: _removedTitle, ...conversationRest }, // Exclude title
+        ...rest
+      } = conversation.value;
+
+      const conversationWithoutMessagesAndTitle = {
+        ...rest,
+        conversation: conversationRest,
+      };
+
       chatService.sendAnalyticsEvent({
         type: ObservabilityAIAssistantTelemetryEventType.ChatFeedback,
         payload: {
-          messageWithFeedback: { message, feedback },
-          conversation: conversation.value,
+          feedback,
+          conversation: conversationWithoutMessagesAndTitle,
         },
       });
     }

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_consolidated_items.tsx

@@ -128,9 +128,7 @@ export function ChatConsolidatedItems({
                 });
               }}
               onEditSubmit={(message) => onEditSubmit(item.message, message)}
-              onFeedbackClick={(feedback) => {
-                onFeedback(item.message, feedback);
-              }}
+              onFeedbackClick={(feedback) => onFeedback(feedback)}
               onRegenerateClick={() => {
                 onRegenerate(item.message);
               }}

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_timeline.tsx

@@ -53,7 +53,7 @@ export interface ChatTimelineProps {
   chatState: ChatState;
   currentUser?: Pick<AuthenticatedUser, 'full_name' | 'username'>;
   onEdit: (message: Message, messageAfterEdit: Message) => void;
-  onFeedback: (message: Message, feedback: Feedback) => void;
+  onFeedback: (feedback: Feedback) => void;
   onRegenerate: (message: Message) => void;
   onSendTelemetry: (eventWithPayload: TelemetryEventTypeWithPayload) => void;
   onStopGenerating: () => void;
@@ -137,9 +137,7 @@ export function ChatTimeline({
             onActionClick={(payload) => {
               onActionClick({ message: item.message, payload });
             }}
-            onFeedbackClick={(feedback) => {
-              onFeedback(item.message, feedback);
-            }}
+            onFeedbackClick={(feedback) => onFeedback(feedback)}
             onRegenerateClick={() => {
               onRegenerate(item.message);
             }}

x-pack/platform/packages/shared/kbn-ai-assistant/src/prompt_editor/prompt_editor.tsx

@@ -125,7 +125,7 @@ export function PromptEditor({
       setMode('prompt');
       onSendTelemetry({
         type: ObservabilityAIAssistantTelemetryEventType.UserSentPromptInChat,
-        payload: { ...message, scopes },
+        payload: { scopes },
       });
     } catch (_) {
       setInnerMessage(oldMessage);

x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/index.ts

@@ -6,12 +6,10 @@
  */
 
 import type { AnalyticsServiceSetup, AnalyticsServiceStart } from '@kbn/core-analytics-browser';
-import { AssistantScope } from '@kbn/ai-assistant-common';
-import type { Message } from '../../common';
 import { chatFeedbackEventSchema, ChatFeedback } from './schemas/chat_feedback';
 import { insightFeedbackEventSchema, InsightFeedback } from './schemas/insight_feedback';
 import { insightResponseEventSchema, InsightResponse } from './schemas/insight_response';
-import { userSentPromptEventSchema } from './schemas/user_sent_prompt';
+import { userSentPromptEventSchema, UserSentPrompt } from './schemas/user_sent_prompt';
 import { ObservabilityAIAssistantTelemetryEventType } from './telemetry_event_type';
 
 const schemas = [
@@ -26,7 +24,7 @@ export type TelemetryEventTypeWithPayload =
   | { type: ObservabilityAIAssistantTelemetryEventType.InsightFeedback; payload: InsightFeedback }
   | {
       type: ObservabilityAIAssistantTelemetryEventType.UserSentPromptInChat;
-      payload: Message & { scopes: AssistantScope[] };
+      payload: UserSentPrompt;
     }
   | {
       type: ObservabilityAIAssistantTelemetryEventType.InsightResponse;

x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/schemas/chat_feedback.ts

@@ -6,33 +6,24 @@
  */
 
 import type { EventTypeOpts } from '@kbn/core/public';
-import type { Message, Conversation } from '../../../common';
+import type { Conversation } from '../../../common';
 import type { Feedback } from '../../components/buttons/feedback_buttons';
 import { ObservabilityAIAssistantTelemetryEventType } from '../telemetry_event_type';
-import { messageSchema } from './common';
 
 export interface ChatFeedback {
-  messageWithFeedback: {
-    message: Message;
-    feedback: Feedback;
+  feedback: Feedback;
+  conversation: Omit<Omit<Conversation, 'messages'>, 'conversation'> & {
+    conversation: Omit<Conversation['conversation'], 'title'>;
   };
-  conversation: Conversation;
 }
 
 export const chatFeedbackEventSchema: EventTypeOpts<ChatFeedback> = {
   eventType: ObservabilityAIAssistantTelemetryEventType.ChatFeedback,
   schema: {
-    messageWithFeedback: {
-      properties: {
-        message: {
-          properties: messageSchema,
-        },
-        feedback: {
-          type: 'text',
-          _meta: {
-            description: 'Whether the user has deemed this response useful or not',
-          },
-        },
+    feedback: {
+      type: 'text',
+      _meta: {
+        description: 'Whether the user has deemed this response useful or not',
       },
     },
     conversation: {
@@ -68,12 +59,6 @@ export const chatFeedbackEventSchema: EventTypeOpts<ChatFeedback> = {
                 description: 'The id of the conversation.',
               },
             },
-            title: {
-              type: 'text',
-              _meta: {
-                description: 'The title of the conversation.',
-              },
-            },
             last_updated: {
               type: 'text',
               _meta: {
@@ -104,15 +89,6 @@ export const chatFeedbackEventSchema: EventTypeOpts<ChatFeedback> = {
             },
           },
         },
-        messages: {
-          type: 'array',
-          items: {
-            properties: messageSchema,
-          },
-          _meta: {
-            description: 'The messages in the conversation.',
-          },
-        },
         labels: {
           type: 'pass_through',
           _meta: {

x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/schemas/insight_feedback.ts

@@ -6,14 +6,11 @@
  */
 
 import type { EventTypeOpts } from '@kbn/core/public';
-import type { Message } from '../../../common';
 import type { Feedback } from '../../components/buttons/feedback_buttons';
 import { ObservabilityAIAssistantTelemetryEventType } from '../telemetry_event_type';
-import { messageSchema } from './common';
 
 export interface InsightFeedback {
   feedback: Feedback;
-  message: Message;
 }
 
 export const insightFeedbackEventSchema: EventTypeOpts<InsightFeedback> = {
@@ -25,8 +22,5 @@ export const insightFeedbackEventSchema: EventTypeOpts<InsightFeedback> = {
         description: 'Whether the user has deemed this response useful or not',
       },
     },
-    message: {
-      properties: messageSchema,
-    },
   },
 };

x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/schemas/user_sent_prompt.ts

@@ -6,11 +6,22 @@
  */
 
 import type { EventTypeOpts } from '@kbn/core/public';
-import type { Message } from '../../../common';
+import { AssistantScope } from '@kbn/ai-assistant-common';
 import { ObservabilityAIAssistantTelemetryEventType } from '../telemetry_event_type';
-import { messageSchema } from './common';
 
-export const userSentPromptEventSchema: EventTypeOpts<Message> = {
+export interface UserSentPrompt {
+  scopes: AssistantScope[];
+}
+
+export const userSentPromptEventSchema: EventTypeOpts<UserSentPrompt> = {
   eventType: ObservabilityAIAssistantTelemetryEventType.UserSentPromptInChat,
-  schema: messageSchema,
+  schema: {
+    scopes: {
+      type: 'array',
+      items: {
+        type: 'text',
+        _meta: { description: 'Scope of the AI Assistant' },
+      },
+    },
+  },
 };

x-pack/platform/plugins/shared/observability_ai_assistant/public/components/insight/insight.tsx

@@ -120,7 +120,6 @@ function ChatContent({
                       type: ObservabilityAIAssistantTelemetryEventType.InsightFeedback,
                       payload: {
                         feedback,
-                        message: lastAssistantResponse,
                       },
                     });
                   }

x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts

@@ -417,30 +417,16 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
 
                   expect(events.length).to.eql(1);
 
-                  const { messageWithFeedback, conversation } = events[0]
+                  const { feedback, conversation } = events[0]
                     .properties as unknown as ChatFeedback;
 
-                  expect(messageWithFeedback.feedback).to.eql('positive');
-                  expect(messageWithFeedback.message.message).to.eql({
-                    content: 'My response',
-                    function_call: {
-                      arguments: '',
-                      name: '',
-                      trigger: 'assistant',
-                    },
-                    role: 'assistant',
-                  });
-
-                  expect(conversation.conversation.title).to.eql('My title');
+                  expect(feedback).to.eql('positive');
                   expect(conversation.namespace).to.eql('default');
                   expect(conversation.public).to.eql(false);
                   expect(conversation.user?.name).to.eql('editor');
 
-                  const { messages } = conversation;
-
-                  expect(messages.length).to.eql(9);
-
-                  expect(messages[0].message.role).to.eql('system');
+                  expect(conversation.conversation).to.not.have.property('title');
+                  expect(conversation).to.not.have.property('messages');
                 });
               });
             });
@@ -492,30 +478,16 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
 
                     expect(events.length).to.eql(2);
 
-                    const { messageWithFeedback, conversation } = events[1]
+                    const { feedback, conversation } = events[1]
                       .properties as unknown as ChatFeedback;
 
-                    expect(messageWithFeedback.feedback).to.eql('positive');
-                    expect(messageWithFeedback.message.message).to.eql({
-                      content:
-                        'Service Level Indicators (SLIs) are quantifiable defined metrics that measure the performance and availability of a service or distributed system.',
-                      function_call: {
-                        arguments: '',
-                        name: '',
-                        trigger: 'assistant',
-                      },
-                      role: 'assistant',
-                    });
-
-                    expect(conversation.conversation.title).to.eql('My old conversation');
+                    expect(feedback).to.eql('positive');
                     expect(conversation.namespace).to.eql('default');
                     expect(conversation.public).to.eql(false);
                     expect(conversation.user?.name).to.eql('editor');
 
-                    const { messages } = conversation;
-
-                    // Verify that data changed after user interaction before being sent to telemetry
-                    expect(messages.length).to.eql(9);
+                    expect(conversation.conversation).to.not.have.property('title');
+                    expect(conversation).to.not.have.property('messages');
                   });
                 });
               });