[8.19] [Obs AI Assistant] Anonymization support (#223351)

x-pack/platform/packages/shared/ai-infra/inference-common/src/chat_complete/events.ts

@@ -35,7 +35,16 @@ export type ChatCompletionMessageEvent<TToolOptions extends ToolOptions = ToolOp
       toolCalls: ToolCallsOf<TToolOptions>['toolCalls'];
     }
   >;
-
+// with unredactions
+export interface ChatCompletionUnredactedMessageEvent<
+  TToolOptions extends ToolOptions = ToolOptions
+> extends ChatCompletionMessageEvent<TToolOptions> {
+  unredactions: Array<{
+    entity: string;
+    class_name: string;
+    hash: string;
+  }>;
+}
 /**
  * Represent a partial tool call present in a chunk event.
  *

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_item.tsx

@@ -27,7 +27,19 @@ import { ChatItemActions } from './chat_item_actions';
 import { ChatItemAvatar } from './chat_item_avatar';
 import { ChatItemContentInlinePromptEditor } from './chat_item_content_inline_prompt_editor';
 import { ChatTimelineItem } from './chat_timeline';
-
+// Helper function to extract plain text from a React node.
+function extractTextFromReactNode(node: React.ReactNode): string {
+  if (typeof node === 'string' || typeof node === 'number') {
+    return node.toString();
+  }
+  if (Array.isArray(node)) {
+    return node.map(extractTextFromReactNode).join('');
+  }
+  if (React.isValidElement(node)) {
+    return extractTextFromReactNode(node.props.children);
+  }
+  return '';
+}
 export interface ChatItemProps extends Omit<ChatTimelineItem, 'message'> {
   onActionClick: ChatActionClickHandler;
   onEditSubmit: (message: Message) => void;
@@ -36,6 +48,7 @@ export interface ChatItemProps extends Omit<ChatTimelineItem, 'message'> {
   onSendTelemetry: (eventWithPayload: TelemetryEventTypeWithPayload) => void;
   onStopGeneratingClick: () => void;
   isConversationOwnedByCurrentUser: boolean;
+  displayContent?: React.ReactNode;
 }
 
 const moreCompactHeaderClassName = css`
@@ -95,6 +108,7 @@ export function ChatItem({
   onRegenerateClick,
   onSendTelemetry,
   onStopGeneratingClick,
+  anonymizedHighlightedContent,
 }: ChatItemProps) {
   const accordionId = useGeneratedHtmlId({ prefix: 'chat' });
 
@@ -135,17 +149,21 @@ export function ChatItem({
     return onEditSubmit(newMessage);
   };
 
+  // extract text if content is not a string.
   const handleCopyToClipboard = () => {
-    navigator.clipboard.writeText(content || '');
+    const copyText = typeof content === 'string' ? content : extractTextFromReactNode(content);
+    navigator.clipboard.writeText(copyText || '');
   };
 
-  let contentElement: React.ReactNode =
+  let contentElement: React.ReactNode;
+  contentElement =
     content || loading || error ? (
       <ChatItemContentInlinePromptEditor
         editing={editing}
         loading={loading}
         functionCall={functionCall}
         content={content}
+        anonymizedHighlightedContent={anonymizedHighlightedContent}
         role={role}
         onSubmit={handleInlineEditSubmit}
         onActionClick={onActionClick}

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_item_content_inline_prompt_editor.tsx

@@ -26,6 +26,7 @@ interface Props {
   onActionClick: ChatActionClickHandler;
   onSendTelemetry: (eventWithPayload: TelemetryEventTypeWithPayload) => void;
   onSubmit: (message: Message) => void;
+  anonymizedHighlightedContent?: React.ReactNode;
 }
 
 const textContainerClassName = css`
@@ -48,6 +49,7 @@ export function ChatItemContentInlinePromptEditor({
   onActionClick,
   onSendTelemetry,
   onSubmit,
+  anonymizedHighlightedContent,
 }: Props) {
   return !editing ? (
     <EuiPanel
@@ -56,7 +58,12 @@ export function ChatItemContentInlinePromptEditor({
       hasShadow={false}
       className={textContainerClassName}
     >
-      <MessageText content={content || ''} loading={loading} onActionClick={onActionClick} />
+      <MessageText
+        content={content || ''}
+        anonymizedHighlightedContent={anonymizedHighlightedContent}
+        loading={loading}
+        onActionClick={onActionClick}
+      />
     </EuiPanel>
   ) : (
     <EuiPanel

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_timeline.tsx

@@ -7,20 +7,23 @@
 
 import React, { type ReactNode, useMemo } from 'react';
 import { css } from '@emotion/css';
-import { EuiCommentList, useEuiTheme } from '@elastic/eui';
+import { EuiCode, EuiCommentList, useEuiTheme } from '@elastic/eui';
 import type { AuthenticatedUser } from '@kbn/security-plugin/common';
 import { omit } from 'lodash';
-import type { Message } from '@kbn/observability-ai-assistant-plugin/common';
 import {
   ChatActionClickPayload,
   ChatState,
   type Feedback,
+  type Message,
   type ObservabilityAIAssistantChatService,
   type TelemetryEventTypeWithPayload,
+  aiAssistantAnonymizationRules,
 } from '@kbn/observability-ai-assistant-plugin/public';
+import { AnonymizationRule } from '@kbn/observability-ai-assistant-plugin/common';
 import { ChatItem } from './chat_item';
 import { ChatConsolidatedItems } from './chat_consolidated_items';
 import { getTimelineItemsfromConversation } from '../utils/get_timeline_items_from_conversation';
+import { useKibana } from '../hooks/use_kibana';
 import { ElasticLlmConversationCallout } from './elastic_llm_conversation_callout';
 import { KnowledgeBaseReindexingCallout } from '../knowledge_base/knowledge_base_reindexing_callout';
 
@@ -44,6 +47,7 @@ export interface ChatTimelineItem
   error?: any;
   message: Message;
   functionCall?: Message['message']['function_call'];
+  anonymizedHighlightedContent?: React.ReactNode;
 }
 
 export interface ChatTimelineProps {
@@ -71,6 +75,34 @@ export interface ChatTimelineProps {
   }) => void;
 }
 
+// helper using detected entity positions to transform user messages into react node to add text highlighting
+function highlightContent(
+  content: string,
+  detectedEntities: Array<{ start_pos: number; end_pos: number; entity: string }>
+): React.ReactNode {
+  // Sort the entities by start position
+  const sortedEntities = [...detectedEntities].sort((a, b) => a.start_pos - b.start_pos);
+  const parts: Array<string | React.ReactNode> = [];
+  let lastIndex = 0;
+  sortedEntities.forEach((entity, index) => {
+    // Add the text before the entity
+    if (entity.start_pos > lastIndex) {
+      parts.push(content.substring(lastIndex, entity.start_pos));
+    }
+    // Wrap the sensitive text in a span with highlight styles
+    parts.push(
+      <EuiCode key={`user-highlight-${index}`}>
+        {content.substring(entity.start_pos, entity.end_pos)}
+      </EuiCode>
+    );
+    lastIndex = entity.end_pos;
+  });
+  // Add any remaining text after the last entity
+  if (lastIndex < content.length) {
+    parts.push(content.substring(lastIndex));
+  }
+  return parts;
+}
 const euiCommentListClassName = css`
   padding-bottom: 32px;
 `;
@@ -103,6 +135,20 @@ export function ChatTimeline({
       display: none;
     }
   `;
+  const {
+    services: { uiSettings },
+  } = useKibana();
+
+  const { anonymizationEnabled } = useMemo(() => {
+    try {
+      const rules = uiSettings?.get<AnonymizationRule[]>(aiAssistantAnonymizationRules);
+      return {
+        anonymizationEnabled: Array.isArray(rules) && rules.some((rule) => rule.enabled),
+      };
+    } catch (e) {
+      return { anonymizationEnabled: false };
+    }
+  }, [uiSettings]);
 
   const items = useMemo(() => {
     const timelineItems = getTimelineItemsfromConversation({
@@ -121,8 +167,13 @@ export function ChatTimeline({
     let currentGroup: ChatTimelineItem[] | null = null;
 
     for (const item of timelineItems) {
+      const { role, content, unredactions } = item.message.message;
       if (item.display.hide || !item) continue;
 
+      if (anonymizationEnabled && role === 'user' && content && unredactions) {
+        item.anonymizedHighlightedContent = highlightContent(content, unredactions);
+      }
+
       if (item.display.collapsed) {
         if (currentGroup) {
           currentGroup.push(item);
@@ -147,6 +198,7 @@ export function ChatTimeline({
     isConversationOwnedByCurrentUser,
     isArchived,
     onActionClick,
+    anonymizationEnabled,
   ]);
 
   return (

x-pack/platform/packages/shared/kbn-ai-assistant/src/utils/builders.ts

@@ -20,6 +20,7 @@ type BuildMessageProps = DeepPartial<Message> & {
       name: string;
       trigger: MessageRole.Assistant | MessageRole.User | MessageRole.Elastic;
     };
+    unredactions?: Message['message']['unredactions'];
   };
 };
 

x-pack/platform/plugins/shared/observability_ai_assistant/common/index.ts

@@ -5,7 +5,13 @@
  * 2.0.
  */
 
-export type { Message, Conversation, KnowledgeBaseEntry, ConversationCreateRequest } from './types';
+export type {
+  Message,
+  Conversation,
+  KnowledgeBaseEntry,
+  ConversationCreateRequest,
+  AnonymizationRule,
+} from './types';
 export {
   KnowledgeBaseEntryRole,
   MessageRole,
@@ -47,6 +53,7 @@ export {
   aiAssistantLogsIndexPattern,
   aiAssistantSimulatedFunctionCalling,
   aiAssistantSearchConnectorIndexPattern,
+  aiAssistantAnonymizationRules,
 } from './ui_settings/settings_keys';
 
 export { concatenateChatCompletionChunks } from './utils/concatenate_chat_completion_chunks';

x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts

@@ -29,11 +29,31 @@ export interface PendingMessage {
   aborted?: boolean;
   error?: any;
 }
+export interface DetectedEntity {
+  entity: string;
+  class_name: string;
+  start_pos: number;
+  end_pos: number;
+  hash: string;
+  type: 'ner' | 'regex';
+}
+
+export type DetectedEntityType = DetectedEntity['type'];
+export interface Unredaction {
+  entity: string;
+  class_name: string;
+  start_pos: number;
+  end_pos: number;
+  type: 'ner' | 'regex';
+}
+
+export type UnredactionType = Unredaction['type'];
 
 export interface Message {
   '@timestamp': string;
   message: {
     content?: string;
+    unredactions?: Unredaction[];
     name?: string;
     role: MessageRole;
     function_call?: {
@@ -160,3 +180,19 @@ export enum ConversationAccess {
   SHARED = 'shared',
   PRIVATE = 'private',
 }
+
+export interface InferenceChunk {
+  chunkText: string;
+  charStartOffset: number;
+}
+
+export interface AnonymizationRule {
+  id: string;
+  entityClass: string;
+  type: 'regex' | 'ner';
+  pattern?: string;
+  enabled: boolean;
+  builtIn: boolean;
+  description?: string;
+  normalize?: boolean;
+}

x-pack/platform/plugins/shared/observability_ai_assistant/common/ui_settings/settings_keys.ts

@@ -14,3 +14,4 @@ export const aiAssistantSimulatedFunctionCalling =
 export const aiAssistantSearchConnectorIndexPattern =
   'observability:aiAssistantSearchConnectorIndexPattern';
 export const aiAssistantPreferredAIAssistantType = 'aiAssistant:preferredAIAssistantType';
+export const aiAssistantAnonymizationRules = 'observability:aiAssistantAnonymizationRules';

x-pack/platform/plugins/shared/observability_ai_assistant/common/utils/anonymization/redaction.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DetectedEntity } from '../../types';
+/** Regex matching object‑hash placeholders (40 hex chars) */
+export const HASH_REGEX = /[0-9a-f]{40}/g;
+/**
+ * Replace each entity span in the original with its hash.
+ */
+
+export function redactEntities(original: string, entities: DetectedEntity[]): string {
+  let redacted = original;
+  entities
+    .slice()
+    .sort((a, b) => b.start_pos - a.start_pos)
+    .forEach((e) => {
+      redacted = redacted.slice(0, e.start_pos) + e.hash + redacted.slice(e.end_pos);
+    });
+
+  return redacted;
+}
+/**
+ * Replace every placeholder in a string with its real value
+ * (taken from `hashMap`).
+ */
+export function unhashString(
+  contentWithHashes: string,
+  hashMap: Map<string, { value: string }>
+): string {
+  return contentWithHashes.replace(HASH_REGEX, (h) => hashMap.get(h)?.value ?? h);
+}