Added Enhancements to Log Ai Insight

x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml

@@ -43,7 +43,6 @@ dependsOn:
   - '@kbn/inference-plugin'
   - '@kbn/server-route-repository'
   - '@kbn/server-route-repository-utils'
-  - '@kbn/std'
   - '@kbn/spaces-plugin'
   - '@kbn/management-settings-ids'
   - '@kbn/logging'

x-pack/solutions/observability/plugins/observability_agent_builder/server/routes/ai_insights/apm_error/generate_error_ai_insight.ts

@@ -10,15 +10,14 @@ import type { Logger } from '@kbn/logging';
 import { MessageRole } from '@kbn/inference-common';
 import type { BoundInferenceClient } from '@kbn/inference-common';
 import dedent from 'dedent';
-import { concat, of } from 'rxjs';
 import type { ObservabilityAgentBuilderDataRegistry } from '../../../data_registry/data_registry';
 import type {
   ObservabilityAgentBuilderCoreSetup,
   ObservabilityAgentBuilderPluginSetupDependencies,
 } from '../../../types';
 import { fetchApmErrorContext } from './fetch_apm_error_context';
 import { getEntityLinkingInstructions } from '../../../agent/register_observability_agent';
-import type { AiInsightResult, ContextEvent } from '../types';
+import { createAiInsightResult, type AiInsightResult } from '../types';
 
 function getErrorAiInsightSystemPrompt({ urlPrefix }: { urlPrefix: string }) {
   return dedent(`
@@ -120,13 +119,5 @@ export async function generateErrorAiInsight({
     stream: true,
   });
 
-  const streamWithContext$ = concat(
-    of<ContextEvent>({ type: 'context', context: errorContext }),
-    events$
-  );
-
-  return {
-    events$: streamWithContext$,
-    context: errorContext,
-  };
+  return createAiInsightResult(errorContext, events$);
 }

x-pack/solutions/observability/plugins/observability_agent_builder/server/routes/ai_insights/get_alert_ai_insights.ts

@@ -12,9 +12,8 @@ import dedent from 'dedent';
 import { compact, isEmpty } from 'lodash';
 import moment from 'moment';
 import type { Observable } from 'rxjs';
-import { concat, of } from 'rxjs';
 import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry';
-import type { AiInsightResult, ContextEvent } from './types';
+import { createAiInsightResult, type AiInsightResult } from './types';
 import type {
   ObservabilityAgentBuilderCoreSetup,
   ObservabilityAgentBuilderPluginSetupDependencies,
@@ -81,12 +80,7 @@ export async function getAlertAiInsight({
     context: relatedContext,
   });
 
-  const streamWithContext$ = concat(
-    of<ContextEvent>({ type: 'context', context: relatedContext }),
-    events$
-  );
-
-  return { events$: streamWithContext$, context: relatedContext };
+  return createAiInsightResult(relatedContext, events$);
 }
 
 // Time window offsets in minutes before alert start

x-pack/solutions/observability/plugins/observability_agent_builder/server/routes/ai_insights/get_log_ai_insights.ts

@@ -4,46 +4,41 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import moment from 'moment';
-import { MessageRole, type InferenceClient } from '@kbn/inference-common';
-import type { IScopedClusterClient, KibanaRequest } from '@kbn/core/server';
-import { safeJsonStringify } from '@kbn/std';
+import type { Observable } from 'rxjs';
+import type { ChatCompletionEvent, InferenceClient } from '@kbn/inference-common';
+import { MessageRole } from '@kbn/inference-common';
+import type { IScopedClusterClient, KibanaRequest, Logger } from '@kbn/core/server';
 import dedent from 'dedent';
-import { concat, of } from 'rxjs';
-import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry';
 import type { ObservabilityAgentBuilderCoreSetup } from '../../types';
-import { getLogDocumentById } from './get_log_document_by_id';
+import { getLogDocumentById, type LogDocument } from './get_log_document_by_id';
+import { getCorrelatedLogsForLogEntry } from '../../tools/get_correlated_logs/handler';
+import { isWarningOrAbove } from '../../utils/warning_and_above_log_filter';
 import { getEntityLinkingInstructions } from '../../agent/register_observability_agent';
-import type { AiInsightResult, ContextEvent } from './types';
+import { createAiInsightResult, type AiInsightResult } from './types';
 
 export interface GetLogAiInsightsParams {
   core: ObservabilityAgentBuilderCoreSetup;
   index: string;
   id: string;
-  dataRegistry: ObservabilityAgentBuilderDataRegistry;
   inferenceClient: InferenceClient;
   connectorId: string;
   request: KibanaRequest;
   esClient: IScopedClusterClient;
+  logger: Logger;
 }
 
 export async function getLogAiInsights({
   core,
   index,
   id,
-  request,
   esClient,
-  dataRegistry,
   inferenceClient,
   connectorId,
+  request,
+  logger,
 }: GetLogAiInsightsParams): Promise<AiInsightResult> {
-  const systemPrompt = dedent(`
-    You are assisting an SRE who is viewing a log entry in the Kibana Logs UI.
-    Using the provided data produce a concise, action-oriented response.
-
-    ${getEntityLinkingInstructions({ urlPrefix: core.http.basePath.get(request) })}
-  `);
-
   const logEntry = await getLogDocumentById({
     esClient: esClient.asCurrentUser,
     index,
@@ -54,6 +49,47 @@ export async function getLogAiInsights({
     throw new Error('Log entry not found');
   }
 
+  const context = await fetchLogContext({
+    core,
+    logger,
+    esClient,
+    index,
+    id,
+    logEntry,
+  });
+
+  const events$ = generateLogSummary({
+    inferenceClient,
+    connectorId,
+    logEntry,
+    context,
+    core,
+    request,
+  });
+
+  return createAiInsightResult(context, events$);
+}
+
+async function fetchLogContext({
+  core,
+  logger,
+  esClient,
+  index,
+  id,
+  logEntry,
+}: {
+  core: ObservabilityAgentBuilderCoreSetup;
+  logger: Logger;
+  esClient: IScopedClusterClient;
+  index: string;
+  id: string;
+  logEntry: LogDocument;
+}): Promise<string> {
+  const logTimestamp = logEntry['@timestamp'];
+  const logTime = moment(logTimestamp);
+  const windowStart = logTime.clone().subtract(60, 'minutes').toISOString();
+  const windowEnd = logTime.clone().add(60, 'minutes').toISOString();
+
   let context = dedent(`
     <LogEntryIndex>
     ${index}
@@ -63,47 +99,97 @@ export async function getLogAiInsights({
     </LogEntryId>
     <LogEntryFields>
     \`\`\`json
-    ${safeJsonStringify(logEntry)}
+    ${JSON.stringify(logEntry, null, 2)}
     \`\`\`
     </LogEntryFields>
   `);
 
-  if (logEntry.service?.name && logEntry.service?.environment) {
-    const serviceSummary = await dataRegistry.getData('apmServiceSummary', {
-      request,
-      serviceName: logEntry.service?.name,
-      serviceEnvironment: logEntry.service?.environment,
-      start: moment(logEntry['@timestamp']).subtract(24, 'hours').toISOString(),
-      end: moment(logEntry['@timestamp']).add(24, 'hours').toISOString(),
+  let correlatedLogsResult;
+  try {
+    const { sequences } = await getCorrelatedLogsForLogEntry({
+      core,
+      logger,
+      esClient,
+      index,
+      start: windowStart,
+      end: windowEnd,
+      logId: id,
     });
+    correlatedLogsResult = sequences[0];
+  } catch (error) {
+    logger.debug(`Failed to fetch correlated logs: ${error.message}`);
+  }
+
+  if (correlatedLogsResult?.logs?.length) {
     context += dedent(`
-      <ServiceSummary>
+      <CorrelatedLogSequence>
+      Time window: ${windowStart} to ${windowEnd}
       \`\`\`json
-      ${safeJsonStringify(serviceSummary)}
+      ${JSON.stringify(correlatedLogsResult, null, 2)}
       \`\`\`
-      </ServiceSummary>
+      </CorrelatedLogSequence>
     `);
   }
 
-  const events$ = inferenceClient.chatComplete({
+  return context;
+}
+
+function generateLogSummary({
+  inferenceClient,
+  connectorId,
+  logEntry,
+  context,
+  core,
+  request,
+}: {
+  inferenceClient: InferenceClient;
+  connectorId: string;
+  logEntry: LogDocument;
+  context: string;
+  core: ObservabilityAgentBuilderCoreSetup;
+  request: KibanaRequest;
+}): Observable<ChatCompletionEvent> {
+  const isErrorOrWarning = isWarningOrAbove(logEntry);
+  const entityLinkingInstructions = getEntityLinkingInstructions({
+    urlPrefix: core.http.basePath.get(request),
+  });
+
+  const systemPrompt = isErrorOrWarning
+    ? dedent(`
+        You are an expert SRE assistant analyzing an ${logEntry['log.level']} log entry. Provide a thorough investigation:
+
+        - **What happened**: Summarize the error in plain language
+        - **Where it originated**: Identify the service, component, or code path
+        - **Root cause analysis**: Use the available context to identify potential causes
+        - **Impact**: Note any affected downstream services or dependencies
+        - **Next steps**: Suggest specific actions for investigation or remediation
+
+        Base your analysis strictly on the provided data.
+
+        ${entityLinkingInstructions}
+      `)
+    : dedent(`
+        You are an expert SRE assistant analyzing an ${logEntry['log.level']} log entry. Keep it concise:
+
+        - Explain what the log message means in context
+        - Identify the source (service, host, container)
+        - If CorrelatedLogSequence is available, use it to provide additional context
+
+        Base your analysis strictly on the provided data. Be specific and reference actual field values.
+
+        ${entityLinkingInstructions}
+      `);
+
+  const userPrompt = dedent(`
+    ${context}
+    Analyze this log entry and generate a summary explaining what it means.
+    Ensure the analysis is grounded in the provided context, and concise.
+  `);
+
+  return inferenceClient.chatComplete({
     connectorId,
     system: systemPrompt,
-    messages: [
-      {
-        role: MessageRole.User,
-        content:
-          dedent(`Explain this log message: what it means, where it is from, whether it is expected, and if it is an issue.
-          ${context}
-          `),
-      },
-    ],
+    messages: [{ role: MessageRole.User, content: userPrompt }],
     stream: true,
   });
-
-  const streamWithContext$ = concat(of<ContextEvent>({ type: 'context', context }), events$);
-
-  return {
-    events$: streamWithContext$,
-    context,
-  };
 }

x-pack/solutions/observability/plugins/observability_agent_builder/server/routes/ai_insights/get_log_document_by_id.ts

@@ -7,14 +7,28 @@
 
 import type { ElasticsearchClient } from '@kbn/core/server';
 
-export interface LogDocument {
+const LOG_DOCUMENT_FIELDS = [
+  '@timestamp',
+  'message',
+  'log.level',
+  'service.name',
+  'trace.id',
+  'span.id',
+  'http.response.status_code',
+  'error.exception.message',
+] as const;
+
+type FieldKeys = (typeof LOG_DOCUMENT_FIELDS)[number];
+
+export type LogDocument = {
+  'log.level'?: string;
   '@timestamp'?: string;
-  service?: {
-    name?: string;
-    environment?: string;
-  };
-  [key: string]: unknown;
-}
+  message?: string;
+  'http.response.status_code'?: number;
+  'error.exception.message'?: string;
+} & {
+  [K in FieldKeys]?: unknown;
+};
 
 export const getLogDocumentById = async ({
   esClient,
@@ -25,10 +39,26 @@ export const getLogDocumentById = async ({
   index: string;
   id: string;
 }): Promise<LogDocument | undefined> => {
-  const result = await esClient.get<LogDocument>({
+  const result = await esClient.search({
     index,
-    id,
+    size: 1,
+    _source: false,
+    fields: [...LOG_DOCUMENT_FIELDS],
+    query: {
+      ids: { values: [id] },
+    },
   });
 
-  return result._source;
+  const hit = result.hits.hits[0];
+
+  if (!hit?.fields) {
+    return undefined;
+  }
+
+  return Object.fromEntries(
+    Object.entries(hit.fields).map(([key, value]) => [
+      key,
+      Array.isArray(value) && value.length === 1 ? value[0] : value,
+    ])
+  ) as LogDocument;
 };

x-pack/solutions/observability/plugins/observability_agent_builder/server/routes/ai_insights/route.ts

@@ -131,7 +131,7 @@ export function getObservabilityAgentBuilderAiInsightsRouteRepository(): ServerR
         id: t.string,
       }),
     }),
-    handler: async ({ request, core, dataRegistry, params, response, logger }) => {
+    handler: async ({ request, core, dataRegistry, params, response, logger, plugins }) => {
       const { index, id } = params.body;
 
       const [coreStart, startDeps] = await core.getStartServices();
@@ -149,7 +149,7 @@ export function getObservabilityAgentBuilderAiInsightsRouteRepository(): ServerR
         connectorId,
         request,
         esClient,
-        dataRegistry,
+        logger,
       });
 
       return response.ok({