[ AutoImport] Introduce automatic log type detection graph

x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts

@@ -455,4 +455,5 @@ export const ecsTestState = {
   packageName: 'testpackage',
   dataStreamName: 'testDataStream',
   formattedSamples: '{"test1": "test1"}',
+  parsedRawSamples: '{"test1": "test1"}',
 };

x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { LogType } from '../../server/constants';
+
+export const logTypeDetectionTestState = {
+  lastExecutedChain: 'testchain',
+  rawSamples: ['{"test1": "test1"}'],
+  exAnswer: 'testanswer',
+  packageName: 'testPackage',
+  dataStreamName: 'testDatastream',
+  finalized: false,
+  logType: LogType.JSON,
+  ecsVersion: 'testVersion',
+};

x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml

@@ -27,7 +27,7 @@ paths:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
                 dataStreamName:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/DataStreamName"
-                rawSamples:
+                encodedRawSamples:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/RawSamples"
                 mapping:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/Mapping"

x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts

@@ -13,15 +13,15 @@ import {
   LangSmithOptions,
   Mapping,
   PackageName,
-  RawSamples,
+  EncodedRawSamples,
 } from '../model/common_attributes';
 import { EcsMappingAPIResponse } from '../model/response_schemas';
 
 export type EcsMappingRequestBody = z.infer<typeof EcsMappingRequestBody>;
 export const EcsMappingRequestBody = z.object({
   packageName: PackageName,
   dataStreamName: DataStreamName,
-  rawSamples: RawSamples,
+  encodedRawSamples: EncodedRawSamples,
   mapping: Mapping.optional(),
   connectorId: Connector,
   langSmithOptions: LangSmithOptions.optional(),

x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml

@@ -16,11 +16,16 @@ components:
       minLength: 1
       description: DataStream name for the integration to be built.
 
+    EncodedRawSamples:
+      type: string
+      format: binary
+      description: Blob containing the RawSamples.
+
     RawSamples:
       type: array
-      items:
+      items: 
         type: string
-      description: String array containing the json raw samples that are used for ecs mapping.
+      description: Array of raw samples.
 
     Mapping:
       type: object

x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts

@@ -22,7 +22,13 @@ export type DataStreamName = z.infer<typeof DataStreamName>;
 export const DataStreamName = z.string().min(1);
 
 /**
- * String array containing the json raw samples that are used for ecs mapping.
+ * Blob containing the RawSamples.
+ */
+export type EncodedRawSamples = z.infer<typeof EncodedRawSamples>;
+export const EncodedRawSamples = z.string();
+
+/**
+ * String array containing the json raw samples.
  */
 export type RawSamples = z.infer<typeof RawSamples>;
 export const RawSamples = z.array(z.string());

x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml

@@ -16,11 +16,14 @@ components:
           required:
             - mapping
             - pipeline
+            - parsedRawSamples
           properties:
             mapping:
               $ref: "./common_attributes.schema.yaml#/components/schemas/Mapping"
             pipeline:
               $ref: "./common_attributes.schema.yaml#/components/schemas/Pipeline"
+            parsedRawSamples:
+              type: string
 
     CategorizationAPIResponse:
       type: object

x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts

@@ -23,6 +23,7 @@ export const EcsMappingAPIResponse = z.object({
   results: z.object({
     mapping: Mapping,
     pipeline: Pipeline,
+    parsedRawSamples: z.string(),
   }),
 });
 

x-pack/plugins/integration_assistant/common/constants.ts

@@ -18,6 +18,7 @@ export const INTEGRATION_ASSISTANT_BASE_PATH = '/api/integration_assistant';
 
 export const ECS_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/ecs`;
 export const CATEGORIZATION_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/categorization`;
+export const LOG_TYPE_DETECTION_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/logtype`;
 export const RELATED_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/related`;
 export const CHECK_PIPELINE_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/pipeline`;
 export const INTEGRATION_BUILDER_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/build`;

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx

@@ -86,7 +86,7 @@ export const useGeneration = ({
         const ecsRequest: EcsMappingRequestBody = {
           packageName: integrationSettings.name ?? '',
           dataStreamName: integrationSettings.dataStreamName ?? '',
-          rawSamples: integrationSettings.logsSampleParsed ?? [],
+          encodedRawSamples: integrationSettings.logsSampleParsed ?? '',
           connectorId: connector.id,
           langSmithOptions: getLangSmithOptions(),
         };
@@ -99,7 +99,11 @@ export const useGeneration = ({
           return;
         }
         const categorizationRequest: CategorizationRequestBody = {
-          ...ecsRequest,
+          packageName: integrationSettings.name ?? '',
+          dataStreamName: integrationSettings.dataStreamName ?? '',
+          rawSamples: JSON.parse(ecsGraphResult.results.parsedRawSamples), // Unstringified JSON
+          connectorId: connector.id,
+          langSmithOptions: getLangSmithOptions(),
           currentPipeline: ecsGraphResult.results.pipeline,
         };
 

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.tsx

@@ -7,74 +7,14 @@
 
 import React, { useCallback, useState } from 'react';
 import { EuiCallOut, EuiFilePicker, EuiFormRow, EuiSpacer, EuiText } from '@elastic/eui';
-import { useKibana } from '@kbn/kibana-react-plugin/public';
-import { isPlainObject } from 'lodash/fp';
 import type { IntegrationSettings } from '../../types';
 import * as i18n from './translations';
 import { useActions } from '../../state';
 
-const MaxLogsSampleRows = 10;
-
-/**
- * Parse the logs sample file content (json or ndjson) and return the parsed logs sample
- */
-const parseLogsContent = (
-  fileContent: string | undefined
-): { error?: string; isTruncated?: boolean; logsSampleParsed?: string[] } => {
-  if (fileContent == null) {
-    return { error: i18n.LOGS_SAMPLE_ERROR.CAN_NOT_READ };
-  }
-  let parsedContent;
-  try {
-    parsedContent = fileContent
-      .split('\n')
-      .filter((line) => line.trim() !== '')
-      .map((line) => JSON.parse(line));
-
-    // Special case for files that can be parsed as both JSON and NDJSON:
-    //   for a one-line array [] -> extract its contents
-    //   for a one-line object {} -> do nothing
-    if (
-      Array.isArray(parsedContent) &&
-      parsedContent.length === 1 &&
-      Array.isArray(parsedContent[0])
-    ) {
-      parsedContent = parsedContent[0];
-    }
-  } catch (parseNDJSONError) {
-    try {
-      parsedContent = JSON.parse(fileContent);
-    } catch (parseJSONError) {
-      return { error: i18n.LOGS_SAMPLE_ERROR.CAN_NOT_PARSE };
-    }
-  }
-
-  if (!Array.isArray(parsedContent)) {
-    return { error: i18n.LOGS_SAMPLE_ERROR.NOT_ARRAY };
-  }
-  if (parsedContent.length === 0) {
-    return { error: i18n.LOGS_SAMPLE_ERROR.EMPTY };
-  }
-
-  let isTruncated = false;
-  if (parsedContent.length > MaxLogsSampleRows) {
-    parsedContent = parsedContent.slice(0, MaxLogsSampleRows);
-    isTruncated = true;
-  }
-
-  if (parsedContent.some((log) => !isPlainObject(log))) {
-    return { error: i18n.LOGS_SAMPLE_ERROR.NOT_OBJECT };
-  }
-
-  const logsSampleParsed = parsedContent.map((log) => JSON.stringify(log));
-  return { isTruncated, logsSampleParsed };
-};
-
 interface SampleLogsInputProps {
   integrationSettings: IntegrationSettings | undefined;
 }
 export const SampleLogsInput = React.memo<SampleLogsInputProps>(({ integrationSettings }) => {
-  const { notifications } = useKibana().services;
   const { setIntegrationSettings } = useActions();
   const [isParsing, setIsParsing] = useState(false);
   const [sampleFileError, setSampleFileError] = useState<string>();
@@ -88,30 +28,36 @@ export const SampleLogsInput = React.memo<SampleLogsInputProps>(({ integrationSe
         return;
       }
 
+      if (logsSampleFile.size > 9000000) {
+        // File size limited to 9 MegaBytes
+        setSampleFileError(i18n.LOGS_SAMPLE_ERROR.LOGS_SAMPLE_FILE_TOO_LARGE);
+        setIntegrationSettings({ ...integrationSettings, logsSampleParsed: undefined });
+        return;
+      }
+
       const reader = new FileReader();
       reader.onload = function (e) {
-        const fileContent = e.target?.result as string | undefined; // We can safely cast to string since we call `readAsText` to load the file.
-        const { error, isTruncated, logsSampleParsed } = parseLogsContent(fileContent);
+        const logsSampleParsed = e.target?.result as string | undefined; // We can safely cast to string since we call `readAsDataURL` to load the file.
         setIsParsing(false);
-        setSampleFileError(error);
-        if (error) {
+
+        if (logsSampleParsed === undefined) {
+          setSampleFileError(i18n.LOGS_SAMPLE_ERROR.EMPTY);
           setIntegrationSettings({ ...integrationSettings, logsSampleParsed: undefined });
           return;
         }
-
-        if (isTruncated) {
-          notifications?.toasts.addInfo(i18n.LOGS_SAMPLE_TRUNCATED(MaxLogsSampleRows));
-        }
+        const base64encodedContent = logsSampleParsed.split('base64,')[1];
+        const logsSampleOriginal = Buffer.from(base64encodedContent, 'base64').toString();
 
         setIntegrationSettings({
           ...integrationSettings,
           logsSampleParsed,
+          logsSampleOriginal: logsSampleOriginal.split('\n'),
         });
       };
       setIsParsing(true);
-      reader.readAsText(logsSampleFile);
+      reader.readAsDataURL(logsSampleFile);
     },
-    [integrationSettings, setIntegrationSettings, notifications?.toasts, setIsParsing]
+    [integrationSettings, setIntegrationSettings, setIsParsing]
   );
   return (
     <EuiFormRow
@@ -136,9 +82,6 @@ export const SampleLogsInput = React.memo<SampleLogsInputProps>(({ integrationSe
               <EuiText size="s" textAlign="center">
                 {i18n.LOGS_SAMPLE_DESCRIPTION}
               </EuiText>
-              <EuiText size="xs" color="subdued" textAlign="center">
-                {i18n.LOGS_SAMPLE_DESCRIPTION_2}
-              </EuiText>
             </>
           }
           onChange={onChangeLogsSample}

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts

@@ -108,12 +108,7 @@ export const LOGS_SAMPLE_DESCRIPTION = i18n.translate(
     defaultMessage: 'Drag and drop a file or Browse files.',
   }
 );
-export const LOGS_SAMPLE_DESCRIPTION_2 = i18n.translate(
-  'xpack.integrationAssistant.step.dataStream.logsSample.description2',
-  {
-    defaultMessage: 'JSON/NDJSON format',
-  }
-);
+
 export const LOGS_SAMPLE_TRUNCATED = (maxRows: number) =>
   i18n.translate('xpack.integrationAssistant.step.dataStream.logsSample.truncatedWarning', {
     values: { maxRows },
@@ -144,6 +139,12 @@ export const LOGS_SAMPLE_ERROR = {
       defaultMessage: 'The logs sample file contains non-object entries',
     }
   ),
+  LOGS_SAMPLE_FILE_TOO_LARGE: i18n.translate(
+    'xpack.integrationAssistant.step.dataStream.logsSample.fileTooLarge',
+    {
+      defaultMessage: 'Logs Sample file is too large. Please upload a file smaller than 9MB.',
+    }
+  ),
 };
 
 export const ANALYZING = i18n.translate('xpack.integrationAssistant.step.dataStream.analyzing', {

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/use_deploy_integration.ts

@@ -58,7 +58,7 @@ export const useDeployIntegration = ({
                 description: integrationSettings.dataStreamDescription ?? '',
                 name: integrationSettings.dataStreamName ?? '',
                 inputTypes: integrationSettings.inputTypes ?? [],
-                rawSamples: integrationSettings.logsSampleParsed ?? [],
+                rawSamples: integrationSettings.logsSampleOriginal ?? [],
                 docs: result.docs ?? [],
                 pipeline: result.pipeline,
               },

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/use_check_pipeline.ts

@@ -38,7 +38,7 @@ export const useCheckPipeline = ({ integrationSettings, customPipeline }: CheckP
       try {
         const parameters: CheckPipelineRequestBody = {
           pipeline: customPipeline,
-          rawSamples: integrationSettings.logsSampleParsed ?? [],
+          rawSamples: integrationSettings.logsSampleOriginal ?? [],
         };
         setIsGenerating(true);
 

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/types.ts

@@ -33,5 +33,6 @@ export interface IntegrationSettings {
   dataStreamDescription?: string;
   dataStreamName?: string;
   inputTypes?: InputType[];
-  logsSampleParsed?: string[];
+  logsSampleParsed?: string;
+  logsSampleOriginal?: string[];
 }

x-pack/plugins/integration_assistant/server/constants.ts

@@ -8,3 +8,12 @@
 export const ROUTE_HANDLER_TIMEOUT = 10 * 60 * 1000; // 10 * 60 seconds = 10 minutes
 export const LANG_CHAIN_TIMEOUT = ROUTE_HANDLER_TIMEOUT - 10_000; // 9 minutes 50 seconds
 export const CONNECTOR_TIMEOUT = LANG_CHAIN_TIMEOUT - 10_000; // 9 minutes 40 seconds
+export const MAX_IMPORT_PAYLOAD_BYTES = 9000000; // 9 MegaBytes
+export enum LogType {
+  JSON = 'json',
+  NDJSON = 'ndjson',
+  STRUCTURED = 'structured',
+  UNSTRUCTURED = 'unstructured',
+  CSV = 'csv',
+  UNSUPPORTED = 'unsupported',
+}

x-pack/plugins/integration_assistant/server/graphs/ecs/graph.ts

@@ -38,6 +38,10 @@ const graphState: StateGraphArgs<EcsMappingState>['channels'] = {
     value: (x: string[], y?: string[]) => y ?? x,
     default: () => [],
   },
+  parsedRawSamples: {
+    value: (x: string, y?: string) => y ?? x,
+    default: () => '',
+  },
   formattedSamples: {
     value: (x: string, y?: string) => y ?? x,
     default: () => '',
@@ -113,6 +117,7 @@ function modelOutput(state: EcsMappingState): Partial<EcsMappingState> {
     results: {
       mapping: state.currentMapping,
       pipeline: currentPipeline,
+      parsedRawSamples: JSON.stringify(state.rawSamples),
     },
   };
 }

x-pack/plugins/integration_assistant/server/graphs/log_type_detection/constants.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const LOG_TYPE_JSON = {
+  log_type: 'structured',
+};

x-pack/plugins/integration_assistant/server/graphs/log_type_detection/detection.test.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FakeLLM } from '@langchain/core/utils/testing';
+import { handleLogTypeDetection } from './detection';
+import type { LogTypeDetectionState } from '../../types';
+import { logTypeDetectionTestState } from '../../../__jest__/fixtures/log_type_detection';
+import {
+  ActionsClientChatOpenAI,
+  ActionsClientSimpleChatModel,
+} from '@kbn/langchain/server/language_models';
+
+const mockLlm = new FakeLLM({
+  response: '{ "log_type": "structured"}',
+}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel;
+
+const testState: LogTypeDetectionState = logTypeDetectionTestState;
+
+describe('Testing log type detection handler', () => {
+  it('handleLogTypeDetection()', async () => {
+    const response = await handleLogTypeDetection(testState, mockLlm);
+    expect(response.logType).toStrictEqual('structured');
+    expect(response.lastExecutedChain).toBe('logTypeDetection');
+  });
+});