[8.15] [ AutoImport] Introduce automatic log type detection graph (#190407)

x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SamplesFormatName } from '../../common/api/model/common_attributes';
+
+export const logFormatDetectionTestState = {
+  lastExecutedChain: 'testchain',
+  logSamples: ['{"test1": "test1"}'],
+  exAnswer: 'testanswer',
+  packageName: 'testPackage',
+  dataStreamName: 'testDatastream',
+  finalized: false,
+  samplesFormat: { name: SamplesFormatName.Values.json },
+  ecsVersion: 'testVersion',
+  results: { test1: 'test1' },
+};

x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml

@@ -0,0 +1,36 @@
+openapi: 3.0.3
+info:
+  title: Auto Import Analyze Logs API endpoint
+  version: "1"
+paths:
+  /api/integration_assistant/analyzelogs:
+    post:
+      summary: Analyzes log samples and processes them.
+      operationId: AnalyzeLogs
+      x-codegen-enabled: false
+      description: Analyzes log samples and processes them
+      tags:
+        - Analyze Logs API
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - logSamples
+                - connectorId
+              properties:
+                logSamples:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/LogSamples"
+                connectorId:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector"
+                langSmithOptions:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/LangSmithOptions"
+      responses:
+        200:
+          description: Indicates a successful call.
+          content:
+            application/json:
+              schema:
+                $ref: "../model/response_schemas.schema.yaml#/components/schemas/AnalyzeLogsAPIResponse"

x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Auto Import Analyze Logs API endpoint
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
+
+import { LogSamples, Connector, LangSmithOptions } from '../model/common_attributes';
+import { AnalyzeLogsAPIResponse } from '../model/response_schemas';
+
+export type AnalyzeLogsRequestBody = z.infer<typeof AnalyzeLogsRequestBody>;
+export const AnalyzeLogsRequestBody = z.object({
+  logSamples: LogSamples,
+  connectorId: Connector,
+  langSmithOptions: LangSmithOptions.optional(),
+});
+export type AnalyzeLogsRequestBodyInput = z.input<typeof AnalyzeLogsRequestBody>;
+
+export type AnalyzeLogsResponse = z.infer<typeof AnalyzeLogsResponse>;
+export const AnalyzeLogsResponse = AnalyzeLogsAPIResponse;

x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml

@@ -16,6 +16,12 @@ components:
       minLength: 1
       description: DataStream name for the integration to be built.
 
+    LogSamples:
+      type: array
+      items:
+        type: string
+      description:  String form of the input logsamples.
+
     RawSamples:
       type: array
       items:
@@ -42,6 +48,10 @@ components:
       enum:
         - ndjson
         - json
+        - csv
+        - structured
+        - unstructured
+        - unsupported
 
     SamplesFormat:
       type: object

x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts

@@ -21,6 +21,12 @@ export const PackageName = z.string().min(1);
 export type DataStreamName = z.infer<typeof DataStreamName>;
 export const DataStreamName = z.string().min(1);
 
+/**
+ * String form of the input logsamples.
+ */
+export type LogSamples = z.infer<typeof LogSamples>;
+export const LogSamples = z.array(z.string());
+
 /**
  * String array containing the json raw samples that are used for ecs mapping.
  */
@@ -49,7 +55,14 @@ export const Docs = z.array(z.object({}).passthrough());
  * The name of the log samples format.
  */
 export type SamplesFormatName = z.infer<typeof SamplesFormatName>;
-export const SamplesFormatName = z.enum(['ndjson', 'json']);
+export const SamplesFormatName = z.enum([
+  'ndjson',
+  'json',
+  'csv',
+  'structured',
+  'unstructured',
+  'unsupported',
+]);
 export type SamplesFormatNameEnum = typeof SamplesFormatName.enum;
 export const SamplesFormatNameEnum = SamplesFormatName.enum;
 

x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml

@@ -66,3 +66,20 @@ components:
           properties:
             docs:
               $ref: "./common_attributes.schema.yaml#/components/schemas/Docs"
+
+    AnalyzeLogsAPIResponse:
+      type: object
+      required:
+        - results
+      properties:
+        results:
+          type: object
+          required:
+            - parsedSamples
+          properties:
+            samplesFormat:
+              $ref: "./common_attributes.schema.yaml#/components/schemas/SamplesFormat"
+            parsedSamples:
+              type: array
+              items:
+                type: string

x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts

@@ -16,7 +16,7 @@
 
 import { z } from 'zod';
 
-import { Docs, Mapping, Pipeline } from './common_attributes';
+import { Docs, Mapping, Pipeline, SamplesFormat } from './common_attributes';
 
 export type EcsMappingAPIResponse = z.infer<typeof EcsMappingAPIResponse>;
 export const EcsMappingAPIResponse = z.object({
@@ -48,3 +48,11 @@ export const CheckPipelineAPIResponse = z.object({
     docs: Docs,
   }),
 });
+
+export type AnalyzeLogsAPIResponse = z.infer<typeof AnalyzeLogsAPIResponse>;
+export const AnalyzeLogsAPIResponse = z.object({
+  results: z.object({
+    samplesFormat: SamplesFormat,
+    parsedSamples: z.array(z.string()),
+  }),
+});

x-pack/plugins/integration_assistant/common/constants.ts

@@ -18,6 +18,7 @@ export const INTEGRATION_ASSISTANT_BASE_PATH = '/api/integration_assistant';
 
 export const ECS_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/ecs`;
 export const CATEGORIZATION_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/categorization`;
+export const ANALYZE_LOGS_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/analyzelogs`;
 export const RELATED_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/related`;
 export const CHECK_PIPELINE_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/pipeline`;
 export const INTEGRATION_BUILDER_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/build`;

x-pack/plugins/integration_assistant/common/index.ts

@@ -15,6 +15,7 @@ export {
 } from './api/check_pipeline/check_pipeline';
 export { EcsMappingRequestBody, EcsMappingResponse } from './api/ecs/ecs_route';
 export { RelatedRequestBody, RelatedResponse } from './api/related/related_route';
+export { AnalyzeLogsRequestBody, AnalyzeLogsResponse } from './api/analyze_logs/analyze_logs_route';
 
 export type {
   DataStream,
@@ -35,4 +36,5 @@ export {
   PLUGIN_ID,
   RELATED_GRAPH_PATH,
   CHECK_PIPELINE_PATH,
+  ANALYZE_LOGS_PATH,
 } from './constants';

x-pack/plugins/integration_assistant/public/common/lib/api.ts

@@ -16,6 +16,8 @@ import type {
   CheckPipelineRequestBody,
   CheckPipelineResponse,
   BuildIntegrationRequestBody,
+  AnalyzeLogsRequestBody,
+  AnalyzeLogsResponse,
 } from '../../../common';
 import {
   INTEGRATION_BUILDER_PATH,
@@ -24,7 +26,7 @@ import {
   RELATED_GRAPH_PATH,
   CHECK_PIPELINE_PATH,
 } from '../../../common';
-import { FLEET_PACKAGES_PATH } from '../../../common/constants';
+import { ANALYZE_LOGS_PATH, FLEET_PACKAGES_PATH } from '../../../common/constants';
 
 export interface EpmPackageResponse {
   response: [{ id: string; name: string }];
@@ -42,6 +44,16 @@ export interface RequestDeps {
   abortSignal: AbortSignal;
 }
 
+export const runAnalyzeLogsGraph = async (
+  body: AnalyzeLogsRequestBody,
+  { http, abortSignal }: RequestDeps
+): Promise<AnalyzeLogsResponse> =>
+  http.post<AnalyzeLogsResponse>(ANALYZE_LOGS_PATH, {
+    headers: defaultHeaders,
+    body: JSON.stringify(body),
+    signal: abortSignal,
+  });
+
 export const runEcsGraph = async (
   body: EcsMappingRequestBody,
   { http, abortSignal }: RequestDeps

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/mocks/state.ts

@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import type { Pipeline, Docs } from '../../../../../common';
+import type { Pipeline, Docs, SamplesFormat } from '../../../../../common';
 import type { Actions, State } from '../state';
 import type { AIConnector } from '../types';
 
-const result: { pipeline: Pipeline; docs: Docs } = {
+const result: { pipeline: Pipeline; docs: Docs; samplesFormat: SamplesFormat } = {
   pipeline: {
     description: 'Pipeline to process my_integration my_data_stream_title logs',
     processors: [
@@ -389,6 +389,7 @@ const result: { pipeline: Pipeline; docs: Docs } = {
       ],
     },
   ],
+  samplesFormat: { name: 'json' },
 };
 
 const rawSamples = [
@@ -419,8 +420,7 @@ export const mockState: State = {
     dataStreamName: 'mocked_datastream_name',
     dataStreamDescription: 'Mocked Data Stream Description',
     inputTypes: ['filestream'],
-    logsSampleParsed: rawSamples,
-    samplesFormat: { name: 'ndjson', multiline: false },
+    logSamples: rawSamples,
   },
   isGenerating: false,
   result,

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/state.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 import { createContext, useContext } from 'react';
-import type { Pipeline, Docs } from '../../../../common';
+import type { Pipeline, Docs, SamplesFormat } from '../../../../common';
 import type { AIConnector, IntegrationSettings } from './types';
 
 export interface State {
@@ -16,6 +16,7 @@ export interface State {
   result?: {
     pipeline: Pipeline;
     docs: Docs;
+    samplesFormat?: SamplesFormat;
   };
 }
 

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.test.tsx

@@ -17,23 +17,27 @@ import { TelemetryEventType } from '../../../../../services/telemetry/types';
 const integrationSettings = mockState.integrationSettings!;
 const connector = mockState.connector!;
 
+const mockAnalyzeLogsResults = {
+  parsedSamples: [{ test: 'analyzeLogsResponse' }],
+  sampleLogsFormat: { name: 'json' },
+};
 const mockEcsMappingResults = { pipeline: { test: 'ecsMappingResponse' }, docs: [] };
 const mockCategorizationResults = { pipeline: { test: 'categorizationResponse' }, docs: [] };
 const mockRelatedResults = { pipeline: { test: 'relatedResponse' }, docs: [] };
+const mockRunAnalyzeLogsGraph = jest.fn((_: unknown) => ({ results: mockAnalyzeLogsResults }));
 const mockRunEcsGraph = jest.fn((_: unknown) => ({ results: mockEcsMappingResults }));
 const mockRunCategorizationGraph = jest.fn((_: unknown) => ({
   results: mockCategorizationResults,
 }));
 const mockRunRelatedGraph = jest.fn((_: unknown) => ({ results: mockRelatedResults }));
 
 const defaultRequest = {
-  packageName: integrationSettings.name ?? '',
-  dataStreamName: integrationSettings.dataStreamName ?? '',
-  rawSamples: integrationSettings.logsSampleParsed ?? [],
   connectorId: connector.id,
+  LangSmithOptions: undefined,
 };
 
 jest.mock('../../../../../common/lib/api', () => ({
+  runAnalyzeLogsGraph: (params: unknown) => mockRunAnalyzeLogsGraph(params),
   runEcsGraph: (params: unknown) => mockRunEcsGraph(params),
   runCategorizationGraph: (params: unknown) => mockRunCategorizationGraph(params),
   runRelatedGraph: (params: unknown) => mockRunRelatedGraph(params),
@@ -74,21 +78,39 @@ describe('GenerationModal', () => {
       expect(result.queryByTestId('generationModal')).toBeInTheDocument();
     });
 
+    it('should call runAnalyzeLogsGraph with correct parameters', () => {
+      expect(mockRunAnalyzeLogsGraph).toHaveBeenCalledWith({
+        ...defaultRequest,
+        logSamples: integrationSettings.logSamples ?? [],
+      });
+    });
+
     it('should call runEcsGraph with correct parameters', () => {
-      expect(mockRunEcsGraph).toHaveBeenCalledWith(defaultRequest);
+      expect(mockRunEcsGraph).toHaveBeenCalledWith({
+        ...defaultRequest,
+        rawSamples: mockAnalyzeLogsResults.parsedSamples,
+        packageName: integrationSettings.name ?? '',
+        dataStreamName: integrationSettings.dataStreamName ?? '',
+      });
     });
 
     it('should call runCategorizationGraph with correct parameters', () => {
       expect(mockRunCategorizationGraph).toHaveBeenCalledWith({
         ...defaultRequest,
         currentPipeline: mockEcsMappingResults.pipeline,
+        rawSamples: mockAnalyzeLogsResults.parsedSamples,
+        packageName: integrationSettings.name ?? '',
+        dataStreamName: integrationSettings.dataStreamName ?? '',
       });
     });
 
     it('should call runRelatedGraph with correct parameters', () => {
       expect(mockRunRelatedGraph).toHaveBeenCalledWith({
         ...defaultRequest,
         currentPipeline: mockCategorizationResults.pipeline,
+        rawSamples: mockAnalyzeLogsResults.parsedSamples,
+        packageName: integrationSettings.name ?? '',
+        dataStreamName: integrationSettings.dataStreamName ?? '',
       });
     });
 
@@ -101,7 +123,7 @@ describe('GenerationModal', () => {
         TelemetryEventType.IntegrationAssistantGenerationComplete,
         {
           sessionId: expect.any(String),
-          sampleRows: integrationSettings.logsSampleParsed?.length ?? 0,
+          sampleRows: integrationSettings.logSamples?.length ?? 0,
           actionTypeId: connector.actionTypeId,
           model: expect.anything(),
           provider: connector.apiProvider ?? 'unknown',
@@ -147,7 +169,7 @@ describe('GenerationModal', () => {
         TelemetryEventType.IntegrationAssistantGenerationComplete,
         {
           sessionId: expect.any(String),
-          sampleRows: integrationSettings.logsSampleParsed?.length ?? 0,
+          sampleRows: integrationSettings.logSamples?.length ?? 0,
           actionTypeId: connector.actionTypeId,
           model: expect.anything(),
           provider: connector.apiProvider ?? 'unknown',