[Obs AI Assistant] Anonymization support

x-pack/platform/packages/shared/ai-infra/inference-common/src/chat_complete/events.ts

@@ -35,7 +35,16 @@ export type ChatCompletionMessageEvent<TToolOptions extends ToolOptions = ToolOp
       toolCalls: ToolCallsOf<TToolOptions>['toolCalls'];
     }
   >;
-
+// with unredactions
+export interface ChatCompletionUnredactedMessageEvent<
+  TToolOptions extends ToolOptions = ToolOptions
+> extends ChatCompletionMessageEvent<TToolOptions> {
+  unredactions: Array<{
+    entity: string;
+    class_name: string;
+    hash: string;
+  }>;
+}
 /**
  * Represent a partial tool call present in a chunk event.
  *

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_item.tsx

@@ -27,7 +27,19 @@ import { ChatItemActions } from './chat_item_actions';
 import { ChatItemAvatar } from './chat_item_avatar';
 import { ChatItemContentInlinePromptEditor } from './chat_item_content_inline_prompt_editor';
 import { ChatTimelineItem } from './chat_timeline';
-
+// Helper function to extract plain text from a React node.
+function extractTextFromReactNode(node: React.ReactNode): string {
+  if (typeof node === 'string' || typeof node === 'number') {
+    return node.toString();
+  }
+  if (Array.isArray(node)) {
+    return node.map(extractTextFromReactNode).join('');
+  }
+  if (React.isValidElement(node)) {
+    return extractTextFromReactNode(node.props.children);
+  }
+  return '';
+}
 export interface ChatItemProps extends Omit<ChatTimelineItem, 'message'> {
   onActionClick: ChatActionClickHandler;
   onEditSubmit: (message: Message) => void;
@@ -36,6 +48,7 @@ export interface ChatItemProps extends Omit<ChatTimelineItem, 'message'> {
   onSendTelemetry: (eventWithPayload: TelemetryEventTypeWithPayload) => void;
   onStopGeneratingClick: () => void;
   isConversationOwnedByCurrentUser: boolean;
+  displayContent?: React.ReactNode;
 }
 
 const moreCompactHeaderClassName = css`
@@ -95,6 +108,7 @@ export function ChatItem({
   onRegenerateClick,
   onSendTelemetry,
   onStopGeneratingClick,
+  anonymizedHighlightedContent,
 }: ChatItemProps) {
   const accordionId = useGeneratedHtmlId({ prefix: 'chat' });
 
@@ -135,17 +149,21 @@ export function ChatItem({
     return onEditSubmit(newMessage);
   };
 
+  // extract text if content is not a string.
   const handleCopyToClipboard = () => {
-    navigator.clipboard.writeText(content || '');
+    const copyText = typeof content === 'string' ? content : extractTextFromReactNode(content);
+    navigator.clipboard.writeText(copyText || '');
   };
 
-  let contentElement: React.ReactNode =
+  let contentElement: React.ReactNode;
+  contentElement =
     content || loading || error ? (
       <ChatItemContentInlinePromptEditor
         editing={editing}
         loading={loading}
         functionCall={functionCall}
         content={content}
+        anonymizedHighlightedContent={anonymizedHighlightedContent}
         role={role}
         onSubmit={handleInlineEditSubmit}
         onActionClick={onActionClick}

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_item_content_inline_prompt_editor.tsx

@@ -26,6 +26,7 @@ interface Props {
   onActionClick: ChatActionClickHandler;
   onSendTelemetry: (eventWithPayload: TelemetryEventTypeWithPayload) => void;
   onSubmit: (message: Message) => void;
+  anonymizedHighlightedContent?: React.ReactNode;
 }
 
 const textContainerClassName = css`
@@ -48,6 +49,7 @@ export function ChatItemContentInlinePromptEditor({
   onActionClick,
   onSendTelemetry,
   onSubmit,
+  anonymizedHighlightedContent,
 }: Props) {
   return !editing ? (
     <EuiPanel
@@ -56,7 +58,12 @@ export function ChatItemContentInlinePromptEditor({
       hasShadow={false}
       className={textContainerClassName}
     >
-      <MessageText content={content || ''} loading={loading} onActionClick={onActionClick} />
+      <MessageText
+        content={content || ''}
+        anonymizedHighlightedContent={anonymizedHighlightedContent}
+        loading={loading}
+        onActionClick={onActionClick}
+      />
     </EuiPanel>
   ) : (
     <EuiPanel

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_timeline.tsx

@@ -7,21 +7,24 @@
 
 import React, { type ReactNode, useMemo } from 'react';
 import { css } from '@emotion/css';
-import { EuiCommentList } from '@elastic/eui';
+import { EuiCode, EuiCommentList } from '@elastic/eui';
 import type { AuthenticatedUser } from '@kbn/security-plugin/common';
 import { omit } from 'lodash';
-import type { Message } from '@kbn/observability-ai-assistant-plugin/common';
 import {
   ChatActionClickPayload,
   ChatState,
   type Feedback,
+  type Message,
   type ObservabilityAIAssistantChatService,
   type TelemetryEventTypeWithPayload,
+  aiAssistantAnonymizationRules,
 } from '@kbn/observability-ai-assistant-plugin/public';
+import { AnonymizationRule } from '@kbn/observability-ai-assistant-plugin/common';
 import type { UseKnowledgeBaseResult } from '../hooks/use_knowledge_base';
 import { ChatItem } from './chat_item';
 import { ChatConsolidatedItems } from './chat_consolidated_items';
 import { getTimelineItemsfromConversation } from '../utils/get_timeline_items_from_conversation';
+import { useKibana } from '../hooks/use_kibana';
 import { ElasticLlmConversationCallout } from './elastic_llm_conversation_callout';
 
 export interface ChatTimelineItem
@@ -44,6 +47,7 @@ export interface ChatTimelineItem
   error?: any;
   message: Message;
   functionCall?: Message['message']['function_call'];
+  anonymizedHighlightedContent?: React.ReactNode;
 }
 
 export interface ChatTimelineProps {
@@ -71,6 +75,34 @@ export interface ChatTimelineProps {
   }) => void;
 }
 
+// helper using detected entity positions to transform user messages into react node to add text highlighting
+function highlightContent(
+  content: string,
+  detectedEntities: Array<{ start_pos: number; end_pos: number; entity: string }>
+): React.ReactNode {
+  // Sort the entities by start position
+  const sortedEntities = [...detectedEntities].sort((a, b) => a.start_pos - b.start_pos);
+  const parts: Array<string | React.ReactNode> = [];
+  let lastIndex = 0;
+  sortedEntities.forEach((entity, index) => {
+    // Add the text before the entity
+    if (entity.start_pos > lastIndex) {
+      parts.push(content.substring(lastIndex, entity.start_pos));
+    }
+    // Wrap the sensitive text in a span with highlight styles
+    parts.push(
+      <EuiCode key={`user-highlight-${index}`}>
+        {content.substring(entity.start_pos, entity.end_pos)}
+      </EuiCode>
+    );
+    lastIndex = entity.end_pos;
+  });
+  // Add any remaining text after the last entity
+  if (lastIndex < content.length) {
+    parts.push(content.substring(lastIndex));
+  }
+  return parts;
+}
 const euiCommentListClassName = css`
   padding-bottom: 32px;
 `;
@@ -98,6 +130,21 @@ export function ChatTimeline({
   onActionClick,
   chatState,
 }: ChatTimelineProps) {
+  const {
+    services: { uiSettings },
+  } = useKibana();
+
+  const { anonymizationEnabled } = useMemo(() => {
+    try {
+      const rules = uiSettings?.get<AnonymizationRule[]>(aiAssistantAnonymizationRules);
+      return {
+        anonymizationEnabled: Array.isArray(rules) && rules.some((rule) => rule.enabled),
+      };
+    } catch (e) {
+      return { anonymizationEnabled: false };
+    }
+  }, [uiSettings]);
+
   const items = useMemo(() => {
     const timelineItems = getTimelineItemsfromConversation({
       conversationId,
@@ -115,8 +162,13 @@ export function ChatTimeline({
     let currentGroup: ChatTimelineItem[] | null = null;
 
     for (const item of timelineItems) {
+      const { role, content, unredactions } = item.message.message;
       if (item.display.hide || !item) continue;
 
+      if (anonymizationEnabled && role === 'user' && content && unredactions) {
+        item.anonymizedHighlightedContent = highlightContent(content, unredactions);
+      }
+
       if (item.display.collapsed) {
         if (currentGroup) {
           currentGroup.push(item);
@@ -141,6 +193,7 @@ export function ChatTimeline({
     isConversationOwnedByCurrentUser,
     isArchived,
     onActionClick,
+    anonymizationEnabled,
   ]);
 
   return (

x-pack/platform/packages/shared/kbn-ai-assistant/src/utils/builders.ts

@@ -20,6 +20,7 @@ type BuildMessageProps = DeepPartial<Message> & {
       name: string;
       trigger: MessageRole.Assistant | MessageRole.User | MessageRole.Elastic;
     };
+    unredactions?: Message['message']['unredactions'];
   };
 };
 

x-pack/platform/plugins/private/observability_ai_assistant_management/common/ui_settings.ts

@@ -11,6 +11,7 @@ import { i18n } from '@kbn/i18n';
 import {
   aiAssistantSimulatedFunctionCalling,
   aiAssistantSearchConnectorIndexPattern,
+  aiAssistantAnonymizationRules,
 } from '@kbn/observability-ai-assistant-plugin/common';
 
 export const uiSettings: Record<string, UiSettingsParams> = {
@@ -57,4 +58,41 @@ export const uiSettings: Record<string, UiSettingsParams> = {
     requiresPageReload: true,
     solution: 'oblt',
   },
+  [aiAssistantAnonymizationRules]: {
+    category: ['observability'],
+    name: i18n.translate(
+      'xpack.observabilityAiAssistantManagement.settingsTab.anonymizationRulesLabel',
+      { defaultMessage: 'Anonymization Rules' }
+    ),
+    value: [], // Default is an empty array, which disables all anonymization rules.
+    description: i18n.translate(
+      'xpack.observabilityAiAssistantManagement.settingsPage.anonymizationRulesDescription',
+      {
+        defaultMessage:
+          'JSON array of anonymization rules. Each rule is an object with properties:\n' +
+          '- id: unique string identifier\n' +
+          '- entityClass: class of entity (e.g., PER, ORG, EMAIL, URL)\n' +
+          '- type: "ner" or "regex"\n' +
+          '- pattern: (for regex rules) the regex string to match\n' +
+          '- enabled: boolean flag to turn the rule on or off\n' +
+          '- builtIn: boolean indicating this is a built‑in rule\n' +
+          '- description: optional human‑readable description\n' +
+          'Default is an empty array, which disables all anonymization rules.',
+      }
+    ),
+    schema: schema.arrayOf(
+      schema.object({
+        id: schema.string(),
+        entityClass: schema.string(),
+        type: schema.oneOf([schema.literal('ner'), schema.literal('regex')]),
+        pattern: schema.string(),
+        enabled: schema.boolean(),
+        builtIn: schema.boolean(),
+        description: schema.maybe(schema.string()),
+      })
+    ),
+    type: 'json',
+    requiresPageReload: true,
+    solution: 'oblt',
+  },
 };

x-pack/platform/plugins/private/observability_ai_assistant_management/public/routes/components/settings_page.test.tsx

@@ -16,7 +16,11 @@ const useKnowledgeBaseMock = useKnowledgeBase as jest.Mock;
 
 describe('Settings Page', () => {
   const appContextValue = {
-    config: { spacesEnabled: true, visibilityEnabled: true, logSourcesEnabled: true },
+    config: {
+      spacesEnabled: true,
+      visibilityEnabled: true,
+      logSourcesEnabled: true,
+    },
     setBreadcrumbs: () => {},
   };
   useKnowledgeBaseMock.mockReturnValue({

x-pack/platform/plugins/private/observability_ai_assistant_management/server/config.ts

@@ -18,5 +18,9 @@ export type ObservabilityAIAssistantManagementConfig = TypeOf<typeof configSchem
 
 export const config: PluginConfigDescriptor<ObservabilityAIAssistantManagementConfig> = {
   schema: configSchema,
-  exposeToBrowser: { logSourcesEnabled: true, spacesEnabled: true, visibilityEnabled: true },
+  exposeToBrowser: {
+    logSourcesEnabled: true,
+    spacesEnabled: true,
+    visibilityEnabled: true,
+  },
 };

x-pack/platform/plugins/private/observability_ai_assistant_management/server/plugin.ts

@@ -6,6 +6,7 @@
  */
 
 import { CoreSetup, CoreStart, Plugin } from '@kbn/core/server';
+import { aiAssistantAnonymizationRules } from '@kbn/observability-ai-assistant-plugin/common';
 import { uiSettings } from '../common/ui_settings';
 
 export type ObservabilityPluginSetup = ReturnType<AiAssistantManagementPlugin['setup']>;
@@ -20,7 +21,8 @@ export class AiAssistantManagementPlugin implements Plugin<ObservabilityPluginSe
   constructor() {}
 
   public setup(core: CoreSetup<PluginStart>, plugins: PluginSetup) {
-    core.uiSettings.register(uiSettings);
+    const { [aiAssistantAnonymizationRules]: anonymizationRules, ...restSettings } = uiSettings;
+    core.uiSettings.register(restSettings);
     return {};
   }
 

x-pack/platform/plugins/shared/observability_ai_assistant/common/index.ts

@@ -5,7 +5,13 @@
  * 2.0.
  */
 
-export type { Message, Conversation, KnowledgeBaseEntry, ConversationCreateRequest } from './types';
+export type {
+  Message,
+  Conversation,
+  KnowledgeBaseEntry,
+  ConversationCreateRequest,
+  AnonymizationRule,
+} from './types';
 export {
   KnowledgeBaseEntryRole,
   MessageRole,
@@ -47,6 +53,7 @@ export {
   aiAssistantLogsIndexPattern,
   aiAssistantSimulatedFunctionCalling,
   aiAssistantSearchConnectorIndexPattern,
+  aiAssistantAnonymizationRules,
 } from './ui_settings/settings_keys';
 
 export { concatenateChatCompletionChunks } from './utils/concatenate_chat_completion_chunks';

x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts

@@ -29,11 +29,31 @@ export interface PendingMessage {
   aborted?: boolean;
   error?: any;
 }
+export interface DetectedEntity {
+  entity: string;
+  class_name: string;
+  start_pos: number;
+  end_pos: number;
+  hash: string;
+  type: 'ner' | 'regex';
+}
+
+export type DetectedEntityType = DetectedEntity['type'];
+export interface Unredaction {
+  entity: string;
+  class_name: string;
+  start_pos: number;
+  end_pos: number;
+  type: 'ner' | 'regex';
+}
+
+export type UnredactionType = Unredaction['type'];
 
 export interface Message {
   '@timestamp': string;
   message: {
     content?: string;
+    unredactions?: Unredaction[];
     name?: string;
     role: MessageRole;
     function_call?: {
@@ -160,3 +180,19 @@ export enum ConversationAccess {
   SHARED = 'shared',
   PRIVATE = 'private',
 }
+
+export interface InferenceChunk {
+  chunkText: string;
+  charStartOffset: number;
+}
+
+export interface AnonymizationRule {
+  id: string;
+  entityClass: string;
+  type: 'regex' | 'ner';
+  pattern?: string;
+  enabled: boolean;
+  builtIn: boolean;
+  description?: string;
+  normalize?: boolean;
+}